client.add_alert

Input Parameters	Type	Default	Description
name	str	None	A name for the alert rule
project_id	str	None	The unique identifier for the project.
model_id	str	None	The unique identifier for the model.
alert_type	fdl.AlertType	None	One of `AlertType.PERFORMANCE`, `AlertType.DATA_DRIFT`, `AlertType.DATA_INTEGRITY`, `AlertType.SERVICE_METRICS`, or `AlertType.STATISTIC`
metric	fdl.Metric	None	When alert_type is `AlertType.SERVICE_METRICS` this should be `Metric.TRAFFIC`. When alert_type is `AlertType.PERFORMANCE`, choose one of the following based on the ML model task: For binary_classfication: `Metric.ACCURACY` `Metric.TPR` `Metric.FPR` `Metric.PRECISION` `Metric.RECALL` `Metric.F1_SCORE` `Metric.ECE` `Metric.AUC` For regression: `Metric.R2` `Metric.MSE` `Metric.MAE` `Metric.MAPE` `Metric.WMAPE` For multi-class classification: `Metric.ACCURACY` `Metric.LOG_LOSS` For ranking: `Metric.MAP` `Metric.MEAN_NDCG` When alert_type is `AlertType.DATA_DRIFT` choose one of the following: `Metric.PSI` `Metric.JSD` When alert_type is `AlertType.DATA_INTEGRITY` choose one of the following: `Metric.RANGE_VIOLATION` `Metric.MISSING_VALUE` `Metric.TYPE_VIOLATION` When alert_type is `AlertType.STATISTIC` choose one of the following: `Metric.AVERAGE` `Metric.SUM` `Metric.FREQUENCY`
bin_size	fdl.BinSize	ONE_DAY	Duration for which the metric value is calculated. Choose one of the following: `BinSize.ONE_HOUR` `BinSize.ONE_DAY` `BinSize.SEVEN_DAYS`
compare_to	fdl.CompareTo	None	Whether the metric value compared against a static value or the same bin from a previous time period. `CompareTo.RAW_VALUE` `CompareTo.TIME_PERIOD`.
compare_period	fdl.ComparePeriod	None	Required only when `CompareTo` is `TIME_PERIOD`. Choose one of the following: `ComparePeriod.ONE_DAY` `ComparePeriod.SEVEN_DAYS` `ComparePeriod.ONE_MONTH` `ComparePeriod.THREE_MONTHS`
priority	fdl.Priority	None	`Priority.LOW` `Priority.MEDIUM` `Priority.HIGH`
warning_threshold	float	None	[Optional] Threshold value to crossing which a warning level severity alert will be triggered. This should be a decimal which represents a percentage (e.g. 0.45).
critical_threshold	float	None	Threshold value to crossing which a critical level severity alert will be triggered. This should be a decimal which represents a percentage (e.g. 0.45).
condition	fdl.AlertCondition	None	Specifies if the rule should trigger if the metric is greater than or less than the thresholds. `AlertCondition.LESSER` `AlertCondition.GREATER`
notifications_config	Dict[str, Dict[str, Any]]	None	[Optional] notifications config object created using helper method build_notifications_config()
columns	List[str]	None	Column names on which alert rule is to be created. Applicable only when alert_type is AlertType.DATA_INTEGRITY or AlertType.DRIFT. When alert type is AlertType.DATA_INTEGRITY, it can take [ANY] to check for all columns.
baseline_id	str	None	Name of the baseline whose histogram is compared against the same derived from current data. When no baseline_id is specified then the default baseline is used. Used only when alert type is `AlertType.DATA_DRIFT`.
segment	str	None	The segment to alert on. See Segments for more details.

📘
Info
The Fiddler client can be used to create a variety of alert rules. Rules can be of Data Drift, Performance, Data Integrity, and Service Metrics types and they can be compared to absolute (compare_to = RAW_VALUE) or to relative values (compare_to = TIME_PERIOD).

# To add a Performance type alert rule which triggers an email notification 
# when precision metric is 5% higher than that from 1 hr bin one day ago.

import fiddler as fdl

notifications_config = client.build_notifications_config(
    emails = "[email protected], [email protected]",
)
client.add_alert_rule(
    name = "perf-gt-5prec-1hr-1d-ago",
    project_id = 'project-a',
    model_id = 'model-a',
    alert_type = fdl.AlertType.PERFORMANCE,
    metric = fdl.Metric.PRECISION,
    bin_size = fdl.BinSize.ONE_HOUR, 
    compare_to = fdl.CompareTo.TIME_PERIOD,
    compare_period = fdl.ComparePeriod.ONE_DAY,
    warning_threshold = 0.05,
    critical_threshold = 0.1,
    condition = fdl.AlertCondition.GREATER,
    priority = fdl.Priority.HIGH,
    notifications_config = notifications_config
)


# To add Data Integrity alert rule which triggers an email notification when 
# published events have more than 5 null values in any 1 hour bin for the _age_ column. 
# Notice compare_to = fdl.CompareTo.RAW_VALUE.

import fiddler as fdl

client.add_alert_rule(
    name = "age-null-1hr",
    project_id = 'project-a',
    model_id = 'model-a',
    alert_type = fdl.AlertType.DATA_INTEGRITY,
    metric = fdl.Metric.MISSING_VALUE,
    bin_size = fdl.BinSize.ONE_HOUR, 
    compare_to = fdl.CompareTo.RAW_VALUE,
    priority = fdl.Priority.HIGH,
    warning_threshold = 5,
    critical_threshold = 10,
    condition = fdl.AlertCondition.GREATER,
    column = "age",
    notifications_config = notifications_config
)

# To add a Data Drift type alert rule which triggers an email notification 
# when PSI metric for 'age' column from an hr is 5% higher than that from 'baseline_name' dataset.

import fiddler as fdl

client.add_baseline(project_id='project-a', 
                    model_id='model-a', 
                    baseline_name='baseline_name', 
                    type=fdl.BaselineType.PRE_PRODUCTION, 
                    dataset_id='dataset-a')

notifications_config = client.build_notifications_config(
    emails = "[email protected], [email protected]",
)

client.add_alert_rule(
    name = "psi-gt-5prec-age-baseline_name",
    project_id = 'project-a',
    model_id = 'model-a',
    alert_type = fdl.AlertType.DATA_DRIFT,
    metric = fdl.Metric.PSI,
    bin_size = fdl.BinSize.ONE_HOUR, 
    compare_to = fdl.CompareTo.RAW_VALUE,
    warning_threshold = 0.05,
    critical_threshold = 0.1,
    condition = fdl.AlertCondition.GREATER,
    priority = fdl.Priority.HIGH,
    notifications_config = notifications_config,
    columns = ["age"],
    baseline_id = 'baseline_name'
)

# To add Drift type alert rule which triggers an email notification when 
# value of JSD metric is more than 0.5 for one hour bin for  _age_ or _gender_ columns. 
# Notice compare_to = fdl.CompareTo.RAW_VALUE.

import fiddler as fdl
notifications_config = client.build_notifications_config(
    emails = "[email protected], [email protected]",
)

client.add_alert_rule(
    name = "jsd_multi_col_1hr",
    project_id = 'project-a',
    model_id = 'model-a',
    alert_type = fdl.AlertType.DATA_DRIFT,
    metric = fdl.Metric.JSD,
    bin_size = fdl.BinSize.ONE_HOUR, 
    compare_to = fdl.CompareTo.RAW_VALUE,
    warning_threshold = 0.4,
    critical_threshold = 0.5,
    condition = fdl.AlertCondition.GREATER,
    priority = fdl.Priority.HIGH,
    notifications_config = notifications_config,
    columns = ["age", "gender"],
)

Return Type	Description
Alert Rule	Created Alert Rule object

Example responses:

[AlertRule(alert_rule_uuid='9b8711fa-735e-4a72-977c-c4c8b16543ae',
           organization_name='some_org_name',
           project_id='project-a',
           model_id='model-a',
           name='perf-gt-5prec-1hr-1d-ago',
           alert_type=AlertType.PERFORMANCE,
           metric=Metric.PRECISION,
           priority=Priority.HIGH,
           compare_to='CompareTo.TIME_PERIOD,
           compare_period=ComparePeriod.ONE_DAY,
           compare_threshold=None,
           raw_threshold=None,
           warning_threshold=0.05,
           critical_threshold=0.1,
           condition=AlertCondition.GREATER,
           bin_size=BinSize.ONE_HOUR)]

AlertRule(alert_rule_uuid='e1aefdd5-ef22-4e81-b869-3964eff8b5cd', 
organization_name='some_org_name', 
project_id='project-a', 
model_id='model-a', 
name='age-null-1hr', 
alert_type=AlertType.DATA_INTEGRITY, 
metric=Metric.MISSING_VALUE, 
column='age', 
priority=Priority.HIGH, 
compare_to=CompareTo.RAW_VALUE, 
compare_period=None, 
warning_threshold=5, 
critical_threshold=10, 
condition=AlertCondition.GREATER,
bin_size=BinSize.ONE_HOUR)

AlertRule(alert_rule_uuid='e1aefdd5-ef22-4e81-b869-3964eff8b5cd', 
organization_name='some_org_name', 
project_id='project-a', 
model_id='model-a', 
name='psi-gt-5prec-age-baseline_name', 
alert_type=AlertType.DATA_DRIFT, 
metric=Metric.PSI, 
priority=Priority.HIGH, 
compare_to=CompareTo.RAW_VALUE, 
compare_period=None, 
warning_threshold=5, 
critical_threshold=10, 
condition=AlertCondition.GREATER,
bin_size=BinSize.ONE_HOUR,
columns=['age'],
baseline_id='baseline_name')

[AlertRule(alert_rule_uuid='9b8711fa-735e-4a72-977c-c4c8b16543ae',
           organization_name='some_org_name',
           project_id='project-a',
           model_id='model-a',
           name='perf-gt-5prec-1hr-1d-ago',
           alert_type=AlertType.DRIFT,
           metric=Metric.JSD,
           priority=Priority.HIGH,
           compare_to='CompareTo.RAW_VALUE,
           compare_period=ComparePeriod.ONE_HOUR,
           compare_threshold=None,
           raw_threshold=None,
           warning_threshold=0.4,
           critical_threshold=0.5,
           condition=AlertCondition.GREATER,
           bin_size=BinSize.ONE_HOUR,
           columns=['age', 'gender'])]

📘Info

📘
Info