Custom metrics
Custom metrics are used to compute and monitor business or performance metrics. This feature allows you to implement your organization’s specialized metrics, expanding on the insights provided by DataRobot’s built-in service health, data drift, and accuracy metrics.
Manage custom metrics
The following sections outline how to manage custom metrics for deployments.
Create custom metric
To create a custom metric, use CustomMetric.create, as shown in the following example. Provide information for all of the required custom metric fields:
from datarobot.models.deployment import CustomMetric
from datarobot.enums import CustomMetricAggregationType, CustomMetricDirectionality
custom_metric = CustomMetric.create(
deployment_id="5c939e08962d741e34f609f0",
name="My custom metric",
units="x",
is_model_specific=True,
aggregation_type=CustomMetricAggregationType.AVERAGE,
directionality=CustomMetricDirectionality.HIGHER_IS_BETTER,
)
To set the baseline value during metric creation, use the following example.
from datarobot.models.deployment import CustomMetric
from datarobot.enums import (
CustomMetricAggregationType,
CustomMetricDirectionality,
CustomMetricBucketTimeStep,
)
custom_metric = CustomMetric.create(
deployment_id="5c939e08962d741e34f609f0",
name="My custom metric 2",
units="y",
baseline_value=12,
is_model_specific=True,
aggregation_type=CustomMetricAggregationType.AVERAGE,
directionality=CustomMetricDirectionality.HIGHER_IS_BETTER,
time_step=CustomMetricBucketTimeStep.HOUR,
)
Define the names of the columns that are used when submitting values from a dataset.
from datarobot.models.deployment import CustomMetric
from datarobot.enums import CustomMetricAggregationType, CustomMetricDirectionality
custom_metric = CustomMetric.create(
deployment_id="5c939e08962d741e34f609f0",
name="My custom metric 3",
units="z",
baseline_value=1000,
is_model_specific=False,
aggregation_type=CustomMetricAggregationType.SUM,
directionality=CustomMetricDirectionality.LOWER_IS_BETTER,
timestamp_column_name="My Timestamp column",
timestamp_format="%d/%m/%y",
value_column_name="My Value column",
sample_count_column_name="My Sample Count column",
)
For batches, see the column configuration below.
from datarobot.models.deployment import CustomMetric
from datarobot.enums import CustomMetricAggregationType, CustomMetricDirectionality
custom_metric = CustomMetric.create(
deployment_id="5c939e08962d741e34f609f0",
name="My custom metric 4",
units="z",
baseline_value=1000,
is_model_specific=False,
aggregation_type=CustomMetricAggregationType.SUM,
directionality=CustomMetricDirectionality.LOWER_IS_BETTER,
batch_column_name="My Batch column",
)
List custom metrics
To list all custom metrics available for a given deployment, use CustomMetric.list.
from datarobot.models.deployment import CustomMetric
custom_metrics = CustomMetric.list(deployment_id="5c939e08962d741e34f609f0")
custom_metrics
>>> [CustomMetric('66015bdda7ba87e66baa09ee' | 'My custom metric 2'),
CustomMetric('66015bdc5f850c5df3aa09f0' | 'My custom metric')]
Retrieve custom metrics
To get a custom metric by unique identifier, use CustomMetric.get.
from datarobot.models.deployment import CustomMetric
custom_metric = CustomMetric.get(
deployment_id="5c939e08962d741e34f609f0", custom_metric_id="65f17bdcd2d66683cdfc1113")
custom_metric
>>> CustomMetric('66015bdc5f850c5df3aa09f0' | 'My custom metric')
Update custom metrics
To retrieve a custom metric by its unique identifier and update it, use CustomMetric.get() and then update().
from datarobot.models.deployment import CustomMetric
from datarobot.enums import CustomMetricAggregationType, CustomMetricDirectionality
custom_metric = CustomMetric.get(
deployment_id="5c939e08962d741e34f609f0", custom_metric_id="65f17bdcd2d66683cdfc1113")
custom_metric.update(
name="Updated custom metric",
units="foo",
baseline_value=-12,
aggregation_type=CustomMetricAggregationType.SUM,
directionality=CustomMetricDirectionality.LOWER_IS_BETTER,
)
Reset the custom metric baseline
To reset the current metric baseline, use unset_baseline(), as shown in the example below.
from datarobot.models.deployment import CustomMetric
custom_metric = CustomMetric.get(
deployment_id="5c939e08962d741e34f609f0", custom_metric_id="65f17bdcd2d66683cdfc1113")
custom_metric.baseline_values
>>> [{'value': -12.0}]
custom_metric.unset_baseline()
custom_metric.baseline_values
>>> []
Delete custom metrics
To delete a custom metric by unique identifier, use CustomMetric.delete, as in the following example:
from datarobot.models.deployment import CustomMetric
CustomMetric.delete(deployment_id="5c939e08962d741e34f609f0", custom_metric_id="65f17bdcd2d66683cdfc1113")
Submit custom metric values
The following sections outline how to submit custom metric values from various sources.
Submit values from JSON
To submit aggregated custom metric values from JSON, use the submit_values method. Submit data in the form of a list of dictionaries.
from datarobot.models.deployment import CustomMetric
custom_metric = CustomMetric.get(
deployment_id="5c939e08962d741e34f609f0", custom_metric_id="65f17bdcd2d66683cdfc1113")
data = [{'value': 12, 'sample_size': 3, 'timestamp': '2024-03-15T18:00:00'},
{'value': 11, 'sample_size': 5, 'timestamp': '2024-03-15T17:00:00'},
{'value': 14, 'sample_size': 3, 'timestamp': '2024-03-15T16:00:00'}]
custom_metric.submit_values(data=data)
# data witch association IDs
data = [{'value': 15, 'sample_size': 2, 'timestamp': '2024-03-15T21:00:00', 'association_id': '65f44d04dbe192b552e752aa'},
{'value': 13, 'sample_size': 6, 'timestamp': '2024-03-15T20:00:00', 'association_id': '65f44d04dbe192b552e753bb'},
{'value': 17, 'sample_size': 2, 'timestamp': '2024-03-15T19:00:00', 'association_id': '65f44d04dbe192b552e754cc'}]
custom_metric.submit_values(data=data)
To submit data in the form of a pandas DataFrame:
from datetime import datetime
import pandas as pd
from datarobot.models.deployment import CustomMetric
df = pd.DataFrame(
data={
"timestamp": [
datetime(year=2024, month=3, day=10),
datetime(year=2024, month=3, day=11),
datetime(year=2024, month=3, day=12),
datetime(year=2024, month=3, day=13),
datetime(year=2024, month=3, day=14),
datetime(year=2024, month=3, day=15),
],
"value": [28, 34, 29, 1, 2, 13],
"sample_size": [1, 2, 3, 4, 1, 2],
}
)
custom_metric.submit_values(data=df)
For deployment-specific metrics, do not provide model information. For model specific metrics set model_package_id or model_id.
custom_metric.submit_values(data=data, model_package_id="6421df32525c58cc6f991f25")
custom_metric.submit_values(data=data, model_id="6444482e5583f6ee2e572265")
Use a dry run to test uploads without saving metric data in DataRobot. This option is disabled by default.
custom_metric.submit_values(data=data, dry_run=True)
To send data for a given segment, it must be specified as follows. Note that more than one segment can be specified.
segments = [{"name": "custom_seg", "value": "baz"}]
custom_metric.submit_values(data=data, segments=segments)
Batch mode requires specifying batch IDs. Batches always specify a model by model_package_id or model_id.
from datarobot.models.deployment import CustomMetric
custom_metric = CustomMetric.get(
deployment_id="5c939e08962d741e34f600e1", custom_metric_id="65f17bdcd2d66683cdfc2224")
data = [{'value': 12, 'sample_size': 3, 'batch': '65f44c93fedc5de16b673aaa'},
{'value': 11, 'sample_size': 5, 'batch': '65f44c93fedc5de16b673bbb'},
{'value': 14, 'sample_size': 3, 'batch': '65f44c93fedc5de16b673ccc'}]
custom_metric.submit_values(data=data, model_package_id="6421df32525c58cc6f991f25")
Submit a single value
To report a single metric value at the current moment, use the submit_single_value method.
View the example below, which uses deployment-specific metrics.
from datarobot.models.deployment import CustomMetric
custom_metric = CustomMetric.get(
deployment_id="5c939e08962d741e34f609f0", custom_metric_id="65f17bdcd2d66683cdfc1113")
custom_metric.submit_single_value(value=16)
For model-specific metrics, set model_package_id or model_id.
from datarobot.models.deployment import CustomMetric
custom_metric = CustomMetric.get(
deployment_id="5c939e08962d741e34f609f0", custom_metric_id="65f17bdcd2d66683cdfc1113")
custom_metric.submit_single_value(value=16, model_package_id="6421df32525c58cc6f991f25")
custom_metric.submit_single_value(value=16, model_id="6444482e5583f6ee2e572265")
Dry run and segments work analogously to report aggregated metric values.
custom_metric.submit_single_value(value=16, dry_run=True)
segments = [{"name": "custom_seg", "value": "boo"}]
custom_metric.submit_single_value(value=16, segments=segments)
The sent value timestamp indicates the time the request was sent; the number of sample values is always 1. This method does not support batch submissions.
Submit values from a dataset
To report aggregated custom metrics values from a dataset in the Data Registry, use the submit_values_from_catalog method.
from datarobot.models.deployment import CustomMetric
custom_metric = CustomMetric.get(
deployment_id="5c939e08962d741e34f609f0", custom_metric_id="65f17bdcd2d66683cdfc1113")
# for deployment specific metrics
custom_metric.submit_values_from_catalog(dataset_id="61093144cabd630828bca321")
# for model specific metrics set model_package_id or model_id
custom_metric.submit_values_from_catalog(
dataset_id="61093144cabd630828bca321",
model_package_id="6421df32525c58cc6f991f25"
)
For segmented analysis, define the name of the column in the dataset and the segment to which it corresponds.
segments = [{"name": "custom_seg", "column": "column_with_segment_values"}]
custom_metric.submit_values_from_catalog(
dataset_id="61093144cabd630828bca321",
model_package_id="6421df32525c58cc6f991f25",
segments=segments
)
For batches, specify the batch IDs in the dataset, or send the entire dataset for a single batch ID.
custom_metric.submit_values_from_catalog(
dataset_id="61093144cabd630828bca432",
model_package_id="6421df32525c58cc6f991f25",
batch_id="65f7f71198c2f234b4cb2f7d"
)
The names of the columns in the dataset should correspond to the names of the columns that were defined in the custom metric. In addition, the format of the timestamps should also be the same as defined in the metric. If the sample size is not specified, it is treated as a 1 sample by default. The following example shows the shape of a dataset saved in the AI catalog.
timestamp |
sample_size |
value |
12/12/22 |
1 |
22 |
13/12/22 |
2 |
23 |
14/12/22 |
3 |
24 |
15/12/22 |
4 |
25 |
The following table shows a sample dataset for batches.
batch |
sample_size |
value |
6572db2c9f9d4ad3b9de33d0 |
1 |
22 |
6572db2c9f9d4ad3b9de33d0 |
2 |
23 |
6572db319f9d4ad3b9de33d9 |
3 |
24 |
6572db319f9d4ad3b9de33d9 |
4 |
25 |
Retrieve custom metric values over time
The following sections outline how to retrieve custom metric values.
Retrieve values over a time period
To retrieve values of a custom metric over a time period, use get_values_over_time.
from datetime import datetime, timedelta
from datarobot.enums import BUCKET_SIZE
from datarobot.models.deployment import CustomMetric
custom_metric = CustomMetric.get(
deployment_id="5c939e08962d741e34f609f0", custom_metric_id="65f17bdcd2d66683cdfc1113")
now = datetime.now()
# specify the time window and bucket size by which results are grouped, the default bucket is 7 days
values_over_time = custom_metric.get_values_over_time(
start=now - timedelta(days=2), end=now, bucket_size=BUCKET_SIZE.P1D)
values_over_time
>>> CustomMetricValuesOverTime('2024-03-21 20:15:00+00:00'- '2024-03-23 20:15:00+00:00')"
values_over_time.bucket_values
>>>{datetime.datetime(2024, 3, 22, 10, 0, tzinfo=tzutc()): 1.0,
>>> datetime.datetime(2024, 3, 22, 11, 0, tzinfo=tzutc()): 123.0}}
values_over_time.bucket_sample_sizes
>>>{datetime.datetime(2024, 3, 22, 10, 0, tzinfo=tzutc()): 1,
>>> datetime.datetime(2024, 3, 22, 11, 0, tzinfo=tzutc()): 1}}
values_over_time.get_buckets_as_dataframe()
>>> start end value sample_size
>>> 0 2024-03-21 00:00:00+00:00 2024-03-22 00:00:00+00:00 1.0 1
>>> 1 2024-03-22 00:00:00+00:00 2024-03-23 00:00:00+00:00 123.0 1
For model-specific metrics, set model_package_id or model_id.
values_over_time = custom_metric.get_values_over_time(
start=now - timedelta(days=1), end=now, model_package_id="6421df32525c58cc6f991f25")
values_over_time = custom_metric.get_values_over_time(
start=now - timedelta(days=1), end=now, model_id="6444482e5583f6ee2e572265")
To retrieve values for a specific segment, specify the segment name and its value:
values_over_time = custom_metric.get_values_over_time(
start=now - timedelta(days=1), end=now, segment_attribute="custom_seg", segment_value="val_1")
Retrieve a summary over a time period
To retrieve a summary of a custom metric over a time period, use the get_summary method.
from datetime import datetime, timedelta
from datarobot.enums import BUCKET_SIZE
from datarobot.models.deployment import CustomMetric
custom_metric = CustomMetric.get(
deployment_id="5c939e08962d741e34f609f0", custom_metric_id="65f17bdcd2d66683cdfc1113")
now = datetime.now()
# specify the time window
summary = custom_metric.get_summary(start=now - timedelta(days=7), end=now)
print(summary)
>> "CustomMetricSummary(2024-03-15 15:52:13.392178+00:00 - 2024-03-22 15:52:13.392168+00:00:
{'id': '65fd9b1c0c1a840bc6751ce0', 'name': 'My custom metric', 'value': 215.0, 'sample_count': 13,
'baseline_value': 12.0, 'percent_change': 24.02})"
For model-specific metrics, set model_package_id or model_id.
summary = custom_metric.get_summary(
start=now - timedelta(days=7), end=now, model_package_id="6421df32525c58cc6f991f25")
summary = custom_metric.get_summary(
start=now - timedelta(days=7), end=now, model_id="6444482e5583f6ee2e572265")
To retrieve a summary for a specific segment, specify the segment name and its value.
summary = custom_metric.get_summary(
start=now - timedelta(days=7), end=now, segment_attribute="custom_seg", segment_value="val_1")
Retrieve values over a batch
To retrieve values of a custom metric over a batch, use get_values_over_batch.
from datarobot.models.deployment import CustomMetric
custom_metric = CustomMetric.get(
deployment_id="5c939e08962d741e34f609f0",
custom_metric_id="65f17bdcd2d66683cdfc1113"
)
# All batch metrics, all model-specific
values_over_batch = custom_metric.get_values_over_batch(model_package_id='6421df32525c58cc6f991f25')
values_over_batch.bucket_values
>>> {'6572db2c9f9d4ad3b9de33d0': 35.0, '6572db2c9f9d4ad3b9de44e1': 105.0}
values_over_batch.bucket_sample_sizes
>>> {'6572db2c9f9d4ad3b9de33d0': 6, '6572db2c9f9d4ad3b9de44e1': 8}
values_over_batch.get_buckets_as_dataframe()
>>> batch_id batch_name value sample_size
>>> 0 6572db2c9f9d4ad3b9de33d0 Batch 1 - 03/26/2024 13:04:46 35.0 6
>>> 1 6572db2c9f9d4ad3b9de44e1 Batch 2 - 03/26/2024 13:06:04 105.0 8
For specific batches, set batch_ids.
values_over_batch = custom_metric.get_values_over_batch(
model_package_id='6421df32525c58cc6f991f25', batch_ids=["65f44c93fedc5de16b673aaa", "65f44c93fedc5de16b673bbb"])
To retrieve values for a specific segment, specify the segment name and its value.
values_over_batch = custom_metric.get_values_over_batch(
model_package_id='6421df32525c58cc6f991f25', segment_attribute="custom_seg", segment_value="val_1")
Retrieve a summary over batch
To retrieve a summary of a custom metric over batch, use get_summary:
from datarobot.models.deployment import CustomMetric
custom_metric = CustomMetric.get(
deployment_id="5c939e08962d741e34f609f0",
custom_metric_id="65f17bdcd2d66683cdfc1113"
)
# All batch metrics, all model-specific
batch_summary = custom_metric.get_batch_summary(model_package_id='6421df32525c58cc6f991f25')
print(batch_summary)
>> CustomMetricBatchSummary({'id': '6605396413434b3a7b74342c', 'name': 'batch metric', 'value': 41.25,
'sample_count': 28, 'baseline_value': 123.0, 'percent_change': -66.46})
For specific batches, set batch_ids.
batch_summary = custom_metric.get_batch_summary(
model_package_id='6421df32525c58cc6f991f25', batch_ids=["65f44c93fedc5de16b673aaa", "65f44c93fedc5de16b673bbb"])
To retrieve values for a specific segment, specify the segment name and its value.
batch_summary = custom_metric.get_batch_summary(
model_package_id='6421df32525c58cc6f991f25', segment_attribute="custom_seg", segment_value="val_1")
Hosted custom metrics
Hosted custom metrics allow you to implement up to 5 of your organization’s specialized metrics in a deployment, uploading the custom metric code to DataRobot and hosting the metric calculation on custom jobs infrastructure. After creation, hosted custom metrics can be reused for other deployments. DataRobot provides a variety of templates for common metrics. These metrics can be used as-is, or as a starting point for user-provided metrics.
The following sections outline how to create a hosted custom metric using the Python API client.
Prerequisites
Import the necessary objects to create a custom metric and initialize the DataRobot client.
import datarobot as dr
from datarobot.enums import HostedCustomMetricsTemplateMetricTypeQueryParams
from datarobot.models.deployment.custom_metrics import HostedCustomMetricTemplate, HostedCustomMetric, \
HostedCustomMetricBlueprint, CustomMetric, MetricTimestampSpoofing, ValueField, SampleCountField, BatchField
from datarobot.models.registry import JobRun
from datarobot.models.registry.job import Job
from datarobot import Deployment
from datarobot.models.runtime_parameters import RuntimeParameterValue
from datarobot.models.types import Schedule
dr.Client(token="<DataRobot API Token>", endpoint="<DataRobot URL>")
gen_ai_deployment_1 = Deployment.get('<Deployment Id>')
List hosted custom metric templates
Before creating a hosted custom metric from a template,
retrieve the LLM metric template to use as the basis of the new metric.
To do this, specify the metric_type and,
because the deployments are LLM models handling Japanese text,
search for the specific metric by name, limiting the search to 1 result.
Store the result in templates for the following steps.
templates = HostedCustomMetricTemplate.list(
search="[JP] Character Count",
metric_type=HostedCustomMetricsTemplateMetricTypeQueryParams.LLM,
limit=1,
offset=0,
)
Create a hosted custom metric
After locating the custom metric template, create the hosted custom metric from that template. This method is a shortcut, combining two steps to create the new custom metric from the retrieved template:
Create a custom job for a hosted custom metric from the template previously retrieved (stored in
templates).Connect the hosted custom metric job to the deployment previously defined (stored in
gen_ai_deployment_1).
Specify both the job name and custom metric name in addition to the template and deployment IDs, as you are creating two objects. Additionally, define the job schedule and the runtime parameter overrides for the deployment.
hosted_custom_metric = HostedCustomMetric.create_from_template(
template_id=templates[0].id,
deployment_id=gen_ai_deployment_1.id,
job_name="Hosted Custom Metric Character Count",
custom_metric_name="Character Count",
job_description="Hosted Custom Metric",
custom_metric_description="LLM Character Count",
baseline_value=10,
timestamp=MetricTimestampSpoofing(
column_name="timestamp",
time_format="%Y-%m-%d %H:%M:%S",
),
value = ValueField(column_name="value"),
sample_count=SampleCountField(column_name='Sample Count'),
batch=BatchField(column_name='Batch'),
schedule=Schedule(
day_of_week=[0],
hour=['*'],
minute=['*'],
day_of_month=[12],
month=[1],
),
parameter_overrides=[RuntimeParameterValue(field_name='DRY_RUN', value="0", type="string")]
)
Once we have created the hosted custom metric, initiate the manual run.
job_run = JobRun.create(
job_id=hosted_custom_metric.custom_job_id
runtime_parameter_values=[
RuntimeParameterValue(field_name='DRY_RUN', value="1", type="string"),
RuntimeParameterValue(field_name='DEPLOYMENT_ID', value=gen_ai_deployment_1.id, type="deployment"),
RuntimeParameterValue(field_name='CUSTOM_METRIC_ID', value=hosted_custom_metric.id, type="customMetric"),
]
)
print(job_run.status)
Manually create hosted custom metrics
You can alternatively create hosted custom metrics in a manual sequenced process. This is useful if you want
to edit the custom metric blueprint before attaching the custom job to the deployment.
When you attach the job to the deployment, most settings are
copied from the blueprint (unless you provide an override).
To create the hosted custom metric manually, first
create a custom job from the template (stored in templates).
job = Job.create_from_custom_metric_gallery_template(
template_id=templates[0].id,
name="Job created from template",
description="Job created from template"
)
Next, retrieve the default blueprint provided by the template, and edit it.
blueprint = HostedCustomMetricBlueprint.get(job.id)
print(f"Original directionality: {blueprint.directionality}")
Then, update the parameters of the custom metric in the blueprint.
updated_blueprint = blueprint.update(
directionality='lowerIsBetter',
units='characters',
type='gauge',
time_step='hour',
is_model_specific=False
)
print(f"Updated directionality: {updated_blueprint.directionality}")
Now, create the hosted custom metric. As in the shortcut method, you can provide the job schedule, runtime parameter overrides, and custom metric parameters specific to this deployment.
another_hosted_custom_metric = HostedCustomMetric.create_from_custom_job(
custom_job_id=job.id,
deployment_id=gen_ai_deployment_1.id,
name="Custom metric created in 2 steps",
)
After creating and configuring the metric, verify that the changes to the blueprint are reflected in the custom metric.
another_custom_metric = CustomMetric.get(custom_metric_id=another_hosted_custom_metric.id, deployment_id=gen_ai_deployment_1.id)
print(f"Directionality of another custom metric: {another_custom_metric.directionality}")
Finally, create a manual job run for the custom metric job.
job_run = JobRun.create(
job_id=job.id,
runtime_parameter_values=[
RuntimeParameterValue(field_name='DRY_RUN', value="1", type="string"),
RuntimeParameterValue(field_name='DEPLOYMENT_ID', value=gen_ai_deployment_1.id, type="deployment"),
RuntimeParameterValue(field_name='CUSTOM_METRIC_ID', value=another_hosted_custom_metric.id, type="customMetric"),
]
)
print(job_run.status)
List hosted custom metrics
To list all hosted custom metrics associated with a custom job, use the following code:
hosted_custom_metrics = HostedCustomMetric.list(deployment_id=hosted_custom_metric.custom_job_id)
for metric in hosted_custom_metrics:
print(metric.name)
Delete hosted custom metrics
In addition, you can delete the hosted custom metric, which removes it from deployment while keeping the job, allowing you to create the metric for another deployment.
hosted_custom_metric.delete()
another_hosted_custom_metric.delete()
If necessary, you can delete the entire custom job. If there are any custom metrics associated with that job, they are also deleted.
job.delete()