fdl.DatasetInfo

For information on how to customize these objects, see Customizing Your Dataset Schema.

columns = [
    fdl.Column(
        name='feature_1',
        data_type=fdl.DataType.FLOAT
    ),
    fdl.Column(
        name='feature_2',
        data_type=fdl.DataType.INTEGER
    ),
    fdl.Column(
        name='feature_3',
        data_type=fdl.DataType.BOOLEAN
    ),
    fdl.Column(
        name='output_column',
        data_type=fdl.DataType.FLOAT
    ),
    fdl.Column(
        name='target_column',
        data_type=fdl.DataType.INTEGER
    )
]

dataset_info = fdl.DatasetInfo(
    display_name='Example Dataset',
    columns=columns
)

fdl.DatasetInfo.from_dataframe

import pandas as pd

df = pd.read_csv('example_dataset.csv')

dataset_info = fdl.DatasetInfo.from_dataframe(df=df, max_inferred_cardinality=100)

fdl.DatasetInfo.from_dict

import pandas as pd

df = pd.read_csv('example_dataset.csv')

dataset_info = fdl.DatasetInfo.from_dataframe(df=df, max_inferred_cardinality=100)

dataset_info_dict = dataset_info.to_dict()

new_dataset_info = fdl.DatasetInfo.from_dict(
    deserialized_json={
        'dataset': dataset_info_dict
    }
)

fdl.DatasetInfo.to_dict

import pandas as pd

df = pd.read_csv('example_dataset.csv')

dataset_info = fdl.DatasetInfo.from_dataframe(df=df, max_inferred_cardinality=100)

dataset_info_dict = dataset_info.to_dict()

{
    'name': 'Example Dataset',
    'columns': [
        {
            'column-name': 'feature_1',
            'data-type': 'float'
        },
        {
            'column-name': 'feature_2',
            'data-type': 'int'
        },
        {
            'column-name': 'feature_3',
            'data-type': 'bool'
        },
        {
            'column-name': 'output_column',
            'data-type': 'float'
        },
        {
            'column-name': 'target_column',
            'data-type': 'int'
        }
    ],
    'files': []
}

fdl.ModelInfo

inputs = [
    fdl.Column(
        name='feature_1',
        data_type=fdl.DataType.FLOAT
    ),
    fdl.Column(
        name='feature_2',
        data_type=fdl.DataType.INTEGER
    ),
    fdl.Column(
        name='feature_3',
        data_type=fdl.DataType.BOOLEAN
    )
]

outputs = [
    fdl.Column(
        name='output_column',
        data_type=fdl.DataType.FLOAT
    )
]

targets = [
    fdl.Column(
        name='target_column',
        data_type=fdl.DataType.INTEGER
    )
]

model_info = fdl.ModelInfo(
    display_name='Example Model',
    input_type=fdl.ModelInputType.TABULAR,
    model_task=fdl.ModelTask.BINARY_CLASSIFICATION,
    inputs=inputs,
    outputs=outputs,
    targets=targets
)

fdl.ModelInfo.from_dataset_info

import pandas as pd

df = pd.read_csv('example_dataset.csv')

dataset_info = fdl.DatasetInfo.from_dataframe(
    df=df
)

model_info = fdl.ModelInfo.from_dataset_info(
    dataset_info=dataset_info,
    features=[
        'feature_1',
        'feature_2',
        'feature_3'
    ],
    outputs=[
        'output_column'
    ],
    target='target_column',
    input_type=fdl.ModelInputType.TABULAR,
    model_task=fdl.ModelTask.BINARY_CLASSIFICATION
)

fdl.ModelInfo.from_dict

import pandas as pd

df = pd.read_csv('example_dataset.csv')

dataset_info = fdl.DatasetInfo.from_dataframe(
    df=df
)

model_info = fdl.ModelInfo.from_dataset_info(
    dataset_info=dataset_info,
    features=[
        'feature_1',
        'feature_2',
        'feature_3'
    ],
    outputs=[
        'output_column'
    ],
    target='target_column',
    input_type=fdl.ModelInputType.TABULAR,
    model_task=fdl.ModelTask.BINARY_CLASSIFICATION
)

model_info_dict = model_info.to_dict()

new_model_info = fdl.ModelInfo.from_dict(
    deserialized_json={
        'model': model_info_dict
    }
)

fdl.ModelInfo.to_dict

import pandas as pd

df = pd.read_csv('example_dataset.csv')

dataset_info = fdl.DatasetInfo.from_dataframe(
    df=df
)

model_info = fdl.ModelInfo.from_dataset_info(
    dataset_info=dataset_info,
    features=[
        'feature_1',
        'feature_2',
        'feature_3'
    ],
    outputs=[
        'output_column'
    ],
    target='target_column',
    input_type=fdl.ModelInputType.TABULAR,
    model_task=fdl.ModelTask.BINARY_CLASSIFICATION
)

model_info_dict = model_info.to_dict()

{
    'name': 'Example Model',
    'input-type': 'structured',
    'model-task': 'binary_classification',
    'inputs': [
        {
            'column-name': 'feature_1',
            'data-type': 'float'
        },
        {
            'column-name': 'feature_2',
            'data-type': 'int'
        },
        {
            'column-name': 'feature_3',
            'data-type': 'bool'
        },
        {
            'column-name': 'target_column',
            'data-type': 'int'
        }
    ],
    'outputs': [
        {
            'column-name': 'output_column',
            'data-type': 'float'
        }
    ],
    'datasets': [],
    'targets': [
        {
            'column-name': 'target_column',
            'data-type': 'int'
        }
    ],
    'custom-explanation-names': []
}

fdl.WeightingParams

Holds weighting information for class imbalanced models which can then be passed into a fdl.ModelInfo object. Please note that the use of weighting params requires the presence of model outputs in the baseline dataset.

import pandas as pd
import sklearn.utils
import fiddler as fdl

df = pd.read_csv('example_dataset.csv')
computed_weight = sklearn.utils.class_weight.compute_class_weight(
        class_weight='balanced',
        classes=np.unique(df[TARGET_COLUMN]),
        y=df[TARGET_COLUMN]
    ).tolist()
weighting_params =  fdl.WeightingParams(class_weight=computed_weight)
dataset_info = fdl.DatasetInfo.from_dataframe(df=df)

model_info = fdl.ModelInfo.from_dataset_info(
    dataset_info=dataset_info,
    features=[
        'feature_1',
        'feature_2',
        'feature_3'
    ],
    outputs=['output_column'],
    target='target_column',
    weighting_params=weighting_params,
    input_type=fdl.ModelInputType.TABULAR,
    model_task=fdl.ModelTask.BINARY_CLASSIFICATION
)

fdl.ModelInputType

model_input_type = fdl.ModelInputType.TABULAR

fdl.ModelTask

Represents supported model tasks

model_task = fdl.ModelTask.BINARY_CLASSIFICATION

fdl.DataType

Represents supported data types.

data_type = fdl.DataType.FLOAT

fdl.Column

Represents a column of a dataset.

column = fdl.Column(
    name='feature_1',
    data_type=fdl.DataType.FLOAT,
    value_range_min=0.0,
    value_range_max=80.0
)

fdl.DeploymentParams

Supported from server version 23.1 and above with Model Deployment feature enabled.

deployment_params = fdl.DeploymentParams(
        image_uri="md-base/python/machine-learning:1.1.0",
        cpu=250,
        memory=512,
  		  replicas=1,
)

📘

What parameters should I set for my model?

Setting the right parameters might not be straightforward and Fiddler is here to help you.

The parameters might vary depending the number of input features used, the pre-processing steps used and the model itself.

This table is helping you defining the right parameters

1. Surrogate Models guide

2. User Uploaded guide

For uploading your artifact model, refer to the table above and increase the memory number, depending on your model framework and complexity. Surrogate models use lightgbm framework.

For example, an NLP model for a TEXT input might need memory set at 1024 or higher and CPU at 1000.

📘

Usage Reference

See the usage with:

• add_model_artifact
• add_model_surrogate
• update_model_artifact
• update_model_surrogate

Check more about the Model Deployment feature set.

fdl.ComparePeriod

Required when compare_to = CompareTo.TIME_PERIOD, this field is used to set when comparing against the same bin for a previous time period. Choose from the following:

import fiddler as fdl

client.add_alert_rule(
    name = "perf-gt-5prec-1hr-1d-ago",
    project_name = 'project-a',
    model_name = 'model-a',
    alert_type = fdl.AlertType.PERFORMANCE, 
    metric = fdl.Metric.PRECISION,
    bin_size = fdl.BinSize.ONE_HOUR, 
    compare_to = fdl.CompareTo.TIME_PERIOD,
    compare_period = fdl.ComparePeriod.ONE_DAY, <----
    warning_threshold = 0.05,
    critical_threshold = 0.1,
    condition = fdl.AlertCondition.GREATER,
    priority = fdl.Priority.HIGH,
    notifications_config = notifications_config
)

[AlertRule(alert_rule_uuid='9b8711fa-735e-4a72-977c-c4c8b16543ae',
           organization_name='some_org_name',
           project_id='project-a',
           model_id='model-a',
           name='perf-gt-5prec-1hr-1d-ago',
           alert_type=AlertType.PERFORMANCE, 
           metric=Metric.PRECISION,
           priority=Priority.HIGH,
           compare_to='CompareTo.TIME_PERIOD,
           compare_period=ComparePeriod.ONE_DAY, <----
           compare_threshold=None,
           raw_threshold=None,
           warning_threshold=0.05,
           critical_threshold=0.1,
           condition=AlertCondition.GREATER,
           bin_size=BinSize.ONE_HOUR)]

fdl.AlertCondition

If condition = fdl.AlertCondition.GREATER/LESSER is specified, and an alert is triggered every time the metric value is greater/lesser than the specified threshold.

import fiddler as fdl

client.add_alert_rule(
    name = "perf-gt-5prec-1hr-1d-ago",
    project_name = 'project-a',
    model_name = 'model-a',
    alert_type = fdl.AlertType.PERFORMANCE, 
    metric = fdl.Metric.PRECISION,
    bin_size = fdl.BinSize.ONE_HOUR, 
    compare_to = fdl.CompareTo.TIME_PERIOD,
    compare_period = fdl.ComparePeriod.ONE_DAY,
    warning_threshold = 0.05,
    critical_threshold = 0.1,
    condition = fdl.AlertCondition.GREATER, <-----
    priority = fdl.Priority.HIGH,
    notifications_config = notifications_config
)

[AlertRule(alert_rule_uuid='9b8711fa-735e-4a72-977c-c4c8b16543ae',
           organization_name='some_org_name',
           project_id='project-a',
           model_id='model-a',
           name='perf-gt-5prec-1hr-1d-ago',
           alert_type=AlertType.PERFORMANCE, <---
           metric=Metric.PRECISION,
           priority=Priority.HIGH,
           compare_to='CompareTo.TIME_PERIOD,
           compare_period=ComparePeriod.ONE_DAY,
           compare_threshold=None,
           raw_threshold=None,
           warning_threshold=0.05,
           critical_threshold=0.1,
           condition=AlertCondition.GREATER, <-----
           bin_size=BinSize.ONE_HOUR)]

fdl.CompareTo

Whether the metric value is to be compared against a static value or the same time bin from a previous time period(set using compare_period[ComparePeriod]).

import fiddler as fdl

client.add_alert_rule(
    name = "perf-gt-5prec-1hr-1d-ago",
    project_name = 'project-a',
    model_name = 'binary_classification_model-a',
    alert_type = fdl.AlertType.PERFORMANCE,
    metric = fdl.Metric.PRECISION,
    bin_size = fdl.BinSize.ONE_HOUR, 
    compare_to = fdl.CompareTo.TIME_PERIOD, <----
    compare_period = fdl.ComparePeriod.ONE_DAY,
    warning_threshold = 0.05,
    critical_threshold = 0.1,
    condition = fdl.AlertCondition.GREATER,
    priority = fdl.Priority.HIGH,
    notifications_config = notifications_config
)

[AlertRule(alert_rule_uuid='9b8711fa-735e-4a72-977c-c4c8b16543ae',
           organization_name='some_org_name',
           project_id='project-a',
           model_id='binary_classification_model-a',
           name='perf-gt-5prec-1hr-1d-ago',
           alert_type=AlertType.PERFORMANCE,
           metric=Metric.PRECISION,
           priority=Priority.HIGH,
           compare_to='CompareTo.TIME_PERIOD, <---
           compare_period=ComparePeriod.ONE_DAY,
           compare_threshold=None,
           raw_threshold=None,
           warning_threshold=0.05,
           critical_threshold=0.1,
           condition=AlertCondition.GREATER,
           bin_size=BinSize.ONE_HOUR)]

fdl.BinSize

This field signifies the durations for which fiddler monitoring calculates the metric values

import fiddler as fdl

client.add_alert_rule(
    name = "perf-gt-5prec-1hr-1d-ago",
    project_name = 'project-a',
    model_name = 'model-a',
    alert_type = fdl.AlertType.PERFORMANCE, 
    metric = fdl.Metric.PRECISION,
    bin_size = fdl.BinSize.ONE_HOUR, <----
    compare_to = fdl.CompareTo.TIME_PERIOD,
    compare_period = fdl.ComparePeriod.ONE_DAY,
    warning_threshold = 0.05,
    critical_threshold = 0.1,
    condition = fdl.AlertCondition.GREATER,
    priority = fdl.Priority.HIGH,
    notifications_config = notifications_config
)

[AlertRule(alert_rule_uuid='9b8711fa-735e-4a72-977c-c4c8b16543ae',
           organization_name='some_org_name',
           project_id='project-a',
           model_id='model-a',
           name='perf-gt-5prec-1hr-1d-ago',
           alert_type=AlertType.PERFORMANCE, 
           metric=Metric.PRECISION,
           priority=Priority.HIGH,
           compare_to='CompareTo.TIME_PERIOD,
           compare_period=ComparePeriod.ONE_DAY,
           compare_threshold=None,
           raw_threshold=None,
           warning_threshold=0.05,
           critical_threshold=0.1,
           condition=AlertCondition.GREATER,
           bin_size=BinSize.ONE_HOUR)] <-----

fdl.Priority

This field can be used to prioritize the alert rules by adding an identifier - low, medium, and high to help users better categorize them on the basis of their importance. Following are the Priority Enums:

import fiddler as fdl

client.add_alert_rule(
    name = "perf-gt-5prec-1hr-1d-ago",
    project_name = 'project-a',
    model_name = 'model-a',
    alert_type = fdl.AlertType.PERFORMANCE, 
    metric = fdl.Metric.PRECISION,
    bin_size = fdl.BinSize.ONE_HOUR, 
    compare_to = fdl.CompareTo.TIME_PERIOD,
    compare_period = fdl.ComparePeriod.ONE_DAY,
    warning_threshold = 0.05,
    critical_threshold = 0.1,
    condition = fdl.AlertCondition.GREATER,
    priority = fdl.Priority.HIGH, <---
    notifications_config = notifications_config
)

[AlertRule(alert_rule_uuid='9b8711fa-735e-4a72-977c-c4c8b16543ae',
           organization_name='some_org_name',
           project_id='project-a',
           model_id='model-a',
           name='perf-gt-5prec-1hr-1d-ago',
           alert_type=AlertType.PERFORMANCE, 
           metric=Metric.PRECISION,
           priority=Priority.HIGH, <----
           compare_to='CompareTo.TIME_PERIOD,
           compare_period=ComparePeriod.ONE_DAY,
           compare_threshold=None,
           raw_threshold=None,
           warning_threshold=0.05,
           critical_threshold=0.1,
           condition=AlertCondition.GREATER,
           bin_size=BinSize.ONE_HOUR)]

fdl.Metric

Following is the list of metrics, with corresponding alert type and model task, for which an alert rule can be created.

import fiddler as fdl

client.add_alert_rule(
    name = "perf-gt-5prec-1hr-1d-ago",
    project_name = 'project-a',
    model_name = 'binary_classification_model-a',
    alert_type = fdl.AlertType.PERFORMANCE,
    metric = fdl.Metric.PRECISION, <----
    bin_size = fdl.BinSize.ONE_HOUR, 
    compare_to = fdl.CompareTo.TIME_PERIOD,
    compare_period = fdl.ComparePeriod.ONE_DAY,
    warning_threshold = 0.05,
    critical_threshold = 0.1,
    condition = fdl.AlertCondition.GREATER,
    priority = fdl.Priority.HIGH,
    notifications_config = notifications_config
)

[AlertRule(alert_rule_uuid='9b8711fa-735e-4a72-977c-c4c8b16543ae',
           organization_name='some_org_name',
           project_id='project-a',
           model_id='binary_classification_model-a',
           name='perf-gt-5prec-1hr-1d-ago',
           alert_type=AlertType.PERFORMANCE,
           metric=Metric.PRECISION, <---
           priority=Priority.HIGH,
           compare_to='CompareTo.TIME_PERIOD,
           compare_period=ComparePeriod.ONE_DAY,
           compare_threshold=None,
           raw_threshold=None,
           warning_threshold=0.05,
           critical_threshold=0.1,
           condition=AlertCondition.GREATER,
           bin_size=BinSize.ONE_HOUR)]

fdl.AlertType

client.add_alert_rule(
    name = "perf-gt-5prec-1hr-1d-ago",
    project_name = 'project-a',
    model_name = 'model-a',
    alert_type = fdl.AlertType.PERFORMANCE, <---
    metric = fdl.Metric.PRECISION,
    bin_size = fdl.BinSize.ONE_HOUR, 
    compare_to = fdl.CompareTo.TIME_PERIOD,
    compare_period = fdl.ComparePeriod.ONE_DAY,
    warning_threshold = 0.05,
    critical_threshold = 0.1,
    condition = fdl.AlertCondition.GREATER,
    priority = fdl.Priority.HIGH,
    notifications_config = notifications_config
)

[AlertRule(alert_rule_uuid='9b8711fa-735e-4a72-977c-c4c8b16543ae',
           organization_name='some_org_name',
           project_id='project-a',
           model_id='model-a',
           name='perf-gt-5prec-1hr-1d-ago',
           alert_type=AlertType.PERFORMANCE, <---
           metric=Metric.PRECISION,
           priority=Priority.HIGH,
           compare_to='CompareTo.TIME_PERIOD,
           compare_period=ComparePeriod.ONE_DAY,
           compare_threshold=None,
           raw_threshold=None,
           warning_threshold=0.05,
           critical_threshold=0.1,
           condition=AlertCondition.GREATER,
           bin_size=BinSize.ONE_HOUR)]

fdl.WindowSize

from fiddler import BaselineType, WindowSize

PROJECT_NAME = 'example_project'
BASELINE_NAME = 'example_rolling'
DATASET_NAME = 'example_validation'
MODEL_NAME = 'example_model'

client.add_baseline(
  project_id=PROJECT_NAME,
  model_id=MODEL_NAME,
  baseline_id=BASELINE_NAME,
  type=BaselineType.ROLLING_PRODUCTION,
  offset=WindowSize.ONE_MONTH, # How far back to set our window
  window_size=WindowSize.ONE_WEEK, # Size of the sliding window
)

fdl.CustomFeatureType

fdl.CustomFeature

This is the base class that all other custom features inherit from. It's flexible enough to accommodate different types of derived features. Note: All of the derived feature classes (e.g., Multivariate, VectorFeature, etc.) inherit from CustomFeature and thus have its properties, in addition to their specific ones.

# use from_columns helper function to generate a custom feature combining multiple numeric columns

feature = fdl.CustomFeature.from_columns(
    name='my_feature',
    columns=['column_1', 'column_2'],
    n_clusters=5
)

fdl.Multivariate

Represents custom features derived from multiple columns.

multivariate_feature = fdl.Multivariate(
    name='multi_feature',
    columns=['column_1', 'column_2']
)

fdl.VectorFeature

Represents custom features derived from a single vector column.

vector_feature = fdl.VectorFeature(
    name='vector_feature',
    column='vector_column'
)

fdl.TextEmbedding

Represents custom features derived from text embeddings.

text_embedding_feature = TextEmbedding(
    name='text_custom_feature',
    source_column='text_column',
    column='text_embedding',
    n_tags=10
)

fdl.ImageEmbedding

Represents custom features derived from image embeddings.

image_embedding_feature = fdl.ImageEmbedding(
    name='image_feature',
    source_column='image_url',
    column='image_embedding',
)

fdl.Enrichment (beta)

• Enrichments are custom features designed to augment data provided in events.
• They add new computed columns to your published data automatically whenever defined.
• The new columns generated are available for querying in analyze, charting, and alerting, similar to any other column.

# Automatically generating embedding for a column named “question”

fdl.ModelInfo.from_dataset_info(
    dataset_info=dataset_info,
    display_name='llm_model',
    model_task=fdl.core_objects.ModelTask.LLM,
    custom_features=[
        fdl.Enrichment(
            name='question_embedding',
            enrichment='embedding',
            columns=['question'],
        ),
        fdl.TextEmbedding(
            name='question_cf',
            source_column='question',
            column='question_embedding',
        ),
    ]
)

Note

Enrichments are disabled by default. To enable them, contact your administrator. Failing to do so will result in an error during the add_model call.

Embedding (beta)

• Create an embedding for a string column using an embedding model.
• Supports Sentence transformers and Encoder/Decoder NLP transformers from Hugging Face.
• To enable set enrichment parameter toembedding.
• For each embedding enrichment, if you want to monitor the embedding vector on fiddler you MUST create a corresponding TextEmbedding using the enrichment’s output column.

Requirements:

• Access to Huggingface inference endpoint - https://api-inference.huggingface.co
• Huggingface API token

Supported Models:

fdl.ModelInfo.from_dataset_info(
    dataset_info=dataset_info,
    display_name='llm_model',
    model_task=fdl.core_objects.ModelTask.LLM,
    custom_features = [
      fdl.Enrichment(
          name='Question Embedding', # name of the enrichment, will be the vector col
          enrichment='embedding', 
          columns=['question'], # only one allowed per embedding enrichment, must be a text column in dataframe
          config={ # optional
            'model_name': ... # default: 'thenlper/gte-base'
            'pooling_method': ... # choose from '{first/last/mean}_token'. Only required if NOT using a sentence transformer
          }
      ),
      fdl.TextEmbedding(
        name='question_cf', # name of the text embedding custom feature
        source_column='question', # source - raw text
        column='Question Embedding', # the name of the vector - outpiut of the embedding enrichment
      ),
    ]
)

The above example will lead to generation of new column

• FDL Question Embedding(vector) : embeddings corresponding to string column question

Note

In the context of Hugging Face models, particularly transformer-based models used for generating embeddings, the pooling_method determines how the model processes the output of its layers to produce a single vector representation for input sequences (like sentences or documents). This is crucial when using these models for tasks like sentence or document embedding, where you need a fixed-size vector representation regardless of the input length.

Centroid Distance (beta)

• Fiddler uses KMeans based system to determine which cluster a particular CustomFeature belongs to.
• This Centroid Distance enrichment calculates the distance from the closest centroid calculated by model monitoring.
• A new numeric column with distances to the closest centroid is added to the events table.
• To enable set enrichment parameter tocentroid_distance.

fdl.ModelInfo.from_dataset_info(
    dataset_info=dataset_info,
    display_name='llm_model',
    model_task=fdl.core_objects.ModelTask.LLM,
    custom_features = [
      fdl.Enrichment(
        name='question_embedding',
        enrichment='embedding',
        columns=['question'],
      ),
      fdl.TextEmbedding(
          name='question_cf',
          source_column='question',
          column='question_embedding',
      ),
      fdl.Enrichment(
        name='Centroid Distance',
        enrichment='centroid_distance',
        columns=['question_cf'],
      ),
    ]
)

The above example will lead to generation of new column

• FDL Centroid Distance (question_embedding)(float) : distance from the nearest K-Means centroid present in
  question_embedding

Note

Does not calculate membership for preproduction data, so you cannot calculate drift.

Personally Identifiable Information (beta)

The PII (Personally Identifiable Information) enrichment is a critical tool designed to detect and flag the presence of sensitive information within textual data. Whether user-entered or system-generated, this enrichment aims to identify instances where PII might be exposed, helping to prevent privacy breaches and the potential misuse of personal data. In an era where digital privacy concerns are paramount, mishandling or unintentionally leaking PII can have serious repercussions, including privacy violations, identity theft, and significant legal and reputational damage.

Regulatory frameworks such as the General Data Protection Regulation (GDPR) in the European Union and the Health Insurance Portability and Accountability Act (HIPAA) in the United States underscore the necessity of safeguarding PII. These laws enforce strict guidelines on the collection, storage, and processing of personal data, emphasizing the need for robust measures to protect sensitive information.

The inadvertent inclusion of PII in datasets used for training or interacting with large language models (LLMs) can exacerbate the risks associated with data privacy. Once exposed to an LLM, sensitive information can be inadvertently learned by the model, potentially leading to wider dissemination of this data beyond intended confines. This scenario underscores the importance of preemptively identifying and removing PII from data before it is processed or shared, particularly in contexts involving AI and machine learning.

To mitigate the risks associated with PII exposure, organizations and developers can integrate the PII enrichment into their data processing workflows. This enrichment operates by scanning text for patterns and indicators of personal information, flagging potentially sensitive data for review or anonymization. By proactively identifying PII, stakeholders can take necessary actions to comply with privacy laws, safeguard individuals' data, and prevent the unintended spread of personal information through AI models and other digital platforms. Implementing PII detection and management practices is not just a legal obligation but a critical component of responsible data stewardship in the digital age.

• To enable set enrichment parameter topii.

Requirements

• Reachability to https://github.com/explosion/spacy-models/releases/download/ to download spacy models as required

List of PII entities

fdl.ModelInfo.from_dataset_info(
    dataset_info=dataset_info,
    display_name='llm_model',
    model_task=fdl.core_objects.ModelTask.LLM,
    custom_features = [
      fdl.Enrichment(
        name='Rag PII',
        enrichment='pii',
        columns=['question'], # one or more columns
        allow_list=['fiddler'], # Optional: list of strings that are white listed
        score_threshold=0.85, # Optional: float value for minimum possible confidence 
      ),
    ]
)

The above example will lead to generation of new columns:

1. FDL Rag PII (question) (bool) : whether any PII was detected
2. FDL Rag PII (question) Matches (str) : what matches in raw text were flagged as potential PII (ex. ‘Douglas MacArthur,Korean’)
3. FDL Rag PII (question) Entities (str) : what entites these matches were tagged as (ex. 'PERSON')

Note

PII enrichment is integrated with Presidio

Evaluate (beta)

• Calculates classic Metrics for evaluating QA results like Bleu, Rouge and Meteor.
• To enable set enrichment parameter toevaluate.
• Make sure the reference_col and prediction_col are set in the configof Enrichment.

Here is a summary of the three evaluation metrics for natural language generation:

fdl.ModelInfo.from_dataset_info(
    dataset_info=dataset_info,
    display_name='llm_model',
    model_task=fdl.core_objects.ModelTask.LLM,
    custom_features = [
      fdl.Enrichment(
        name='QA Evaluate',
        enrichment='evaluate',
        columns=['correct_answer', 'generated_answer'],
        config={
            'reference_col': 'correct_answer', # required
            'prediction_col': 'generated_answer', # required
            'metrics': ..., # optional, default - ['bleu', 'rouge' , 'meteor']
        }
      ),
    ]
)

The above example generates 6 new columns

• FDL QA Evaluate (bleu)(float)
• FDL QA Evaluate (rouge1)(float)
• FDL QA Evaluate (rouge2)(float)
• FDL QA Evaluate (rougel)(float)
• FDL QA Evaluate (rougelsum)(float)
• FDL QA Evaluate (meteor)(float)

Textstat (beta)

Generates statistics on string columns.

• To enable set enrichment parameter totextstat.

Supported Statistics

fdl.ModelInfo.from_dataset_info(
    dataset_info=dataset_info,
    display_name='llm_model',
    model_task=fdl.core_objects.ModelTask.LLM,
    custom_features = [
      fdl.Enrichment(
          name='Text Statistics',
          enrichment='textstat',
          columns=['question'],
          config={
          'statistics' : [
              'char_count',
              'dale_chall_readability_score',
            ]
          },
      ),
    ]
)

The above example leads to the creation of two additional columns

• FDL Text Statistics (question) char_count(int) : character count of string in
  questioncolumn
• FDL Text Statistics (question) dale_chall_readability_score(float) : readability score of string in
  question column

Sentiment (beta)

Sentiment Analysis enrichment employs advanced natural language processing (NLP) techniques to gauge the emotional tone behind a body of text. This enrichment is designed to determine whether the sentiment of textual content is positive, negative, or neutral, providing valuable insights into the emotions and opinions expressed within. By analyzing the sentiment, this tool offers a powerful means to understand user feedback, market research responses, social media commentary, and any textual data where opinion and mood are significant.

Implementing Sentiment Analysis into your data processing allows for a nuanced understanding of how your audience feels about a product, service, or topic, enabling informed decision-making and strategy development. It's particularly useful in customer service and brand management, where gauging customer sentiment is crucial for addressing concerns, improving user experience, and building brand reputation.

The Sentiment enrichment uses NLTK's VADER lexicon to generate a score and corresponding sentiment for all specified columns. For each string column on which sentiment enrichment is enabled, two additional columns are added. To enable set enrichment parameter tosentiment.

Requirements

• Reachability to www.nltk.org/nltk_data to download latest vader lexicon

fdl.ModelInfo.from_dataset_info(
    dataset_info=dataset_info,
    display_name='llm_model',
    model_task=fdl.core_objects.ModelTask.LLM,
    custom_features = [
      fdl.Enrichment(
          name='Question Sentiment',
          enrichment='sentiment',
          columns=['question'],
      ),
    ]
)

The above example leads to creation of two columns -

• FDL Question Sentiment (question) compound(float): raw score of sentiment
• FDL Question Sentiment (question) sentiment(string): one of positive, negative and neutral

Profanity (beta)

The Profanity enrichment is designed to detect and flag the use of offensive or inappropriate language within textual content. This enrichment is essential for maintaining the integrity and professionalism of digital platforms, forums, social media, and any user-generated content areas. It helps ensure that conversations and interactions remain respectful and free from language that could be considered harmful or offensive to users.

In the digital space, where diverse audiences come together, the presence of profanity can lead to negative user experiences, damage brand reputation, and create an unwelcoming environment. Implementing a profanity filter is a proactive measure to prevent such outcomes, promoting a positive and inclusive online community.

Beyond maintaining community standards, the Profanity enrichment has practical implications for compliance with platform guidelines and legal regulations concerning hate speech and online conduct. Many digital platforms have strict policies against the use of profane or offensive language, making it crucial for content creators and moderators to actively monitor and manage such language.

By integrating the Profanity enrichment into their content moderation workflow, businesses and content managers can automate the detection of inappropriate language, significantly reducing manual review efforts. This enrichment not only helps in upholding community guidelines and legal standards but also supports the creation of safer and more respectful online spaces for all users.

• Profanity enrichment works off a list of profane terms created from the following two sources
  • The Obscenity List from https://github.com/surge-ai/profanity/blob/main/profanity_en.csv
  • Google banned words https://github.com/coffee-and-fun/google-profanity-words/blob/main/data/en.txt
• To enable set enrichment parameter toprofanityand make sure to specify output_column_name in the config as shown in the example below.

fdl.ModelInfo.from_dataset_info(
    dataset_info=dataset_info,
    display_name='llm_model',
    model_task=fdl.core_objects.ModelTask.LLM,
    custom_features = [
      fdl.Enrichment(
            name='Profanity',
            enrichment='profanity',
            columns=['prompt', 'response'],
            config={'output_column_name': 'contains_profanity'},
        ),
    ]
)

The above example leads to creation of two columns -

• FDL Profanity (prompt) contains_profanity(bool): to indicate if input contains profanity in the value of the prompt column
• FDL Profanity (response) contains_profanity(bool): to indicate if input contains profanity in the value of the response column

Answer Relevance (beta)

The Answer Relevance is a specialized enrichment designed to evaluate the pertinence of AI-generated responses to their corresponding prompts. This enrichment operates by assessing whether the content of a response accurately addresses the question or topic posed by the initial prompt, providing a simple yet effective binary outcome: relevant or not relevant. Its primary function is to ensure that the output of AI systems, such as chatbots, virtual assistants, and content generation models, remains aligned with the user's informational needs and intentions.

In the context of AI-generated content, ensuring relevance is crucial for maintaining user engagement and trust. Irrelevant or tangentially related responses can lead to user frustration, decreased satisfaction, and diminished trust in the AI's capabilities. The Answer Relevance metric serves as a critical checkpoint, verifying that interactions and content deliveries meet the expected standards of accuracy and pertinence.

This enrichment finds its application across a wide range of AI-driven platforms and services where the quality of the response directly impacts the user experience. From customer service bots answering inquiries to educational tools providing study assistance, the ability to automatically gauge the relevance of responses enhances the effectiveness and reliability of these services.

Incorporating the Answer Relevance enrichment into the development and refinement of AI models enables creators to iteratively improve their systems based on relevant feedback. By identifying instances where the model generates non-relevant responses, developers can adjust and fine-tune their models to better meet user expectations. This continuous improvement cycle is essential for advancing the quality and utility of AI-generated content, ensuring that it remains focused, accurate, and highly relevant to users' needs.

• To enable set enrichment parameter toanswer_relevance.

Requirements:

• This enrichment requires access to the OpenAI API, which may introduce latency due to network communication and processing time. Learn more about LLM based enrichments
• OpenAI API access token MUST BE provided by the user.

answer_relevance_config = {
  'prompt' : 'prompt_col',
  'response' : 'response_col',
}

fdl.ModelInfo.from_dataset_info(
    dataset_info=dataset_info,
    display_name='llm_model',
    model_task=fdl.core_objects.ModelTask.LLM,
    custom_features = [
      fdl.Enrichment(
          name = 'Answer Relevance',
          enrichment = 'answer_relevance', 
          columns = ['prompt_col', 'response_col'],
          config = answer_relevance_config,
      ),
    ]
)

The above example will lead to the generation of a new column

• FDL Answer Relevance: Binary metric, which is True if response is relevant to the prompt

Faithfulness (beta)

The Faithfulness (Groundedness) enrichment is a binary indicator designed to evaluate the accuracy and reliability of facts presented in AI-generated text responses. It specifically assesses whether the information used in the response correctly aligns with and is grounded in the provided context, often in the form of referenced documents or data. This enrichment plays a critical role in ensuring that the AI's outputs are not only relevant but also factually accurate, based on the context it was given.

In practical applications, such as automated content creation, customer support, and informational queries, the Faithfulness (Groundedness) metric serves as a safeguard against the dissemination of misinformation. It verifies that the AI system's responses are not only generated with a high level of linguistic fluency but also reflect a true and correct use of the available information (retrieved documents).

This enrichment is particularly important in fields where accuracy is paramount, such as in educational content, medical advice, or factual reporting. By implementing the Faithfulness (Groundedness )metric, developers and researchers can enhance the trustworthiness of AI-generated content, ensuring that users receive responses that are not only contextually relevant but also factually sound. The effectiveness of this enrichment hinges on its ability to critically analyze the alignment between the generated content and the context provided, promoting a higher standard of reliability in AI-generated outputs.

• To enable set enrichment parameter tofaithfulness.

Requirements:

• This enrichment requires access to the OpenAI API, which may introduce latency due to network communication and processing time. Learn more about LLM based enrichments
• OpenAI API access token MUST BE provided by the user.

faithfulness_config = {
  'context' : ['doc_0', 'doc_1', 'doc_2'],
  'response' : 'response_col',
}

fdl.ModelInfo.from_dataset_info(
    dataset_info=dataset_info,
    display_name='llm_model',
    model_task=fdl.core_objects.ModelTask.LLM,
    custom_features = [
      fdl.Enrichment(
          name = 'Faithfulness',
          enrichment = 'faithfulness', 
          columns = ['doc_0', 'doc_1', 'doc_2', 'response_col'],
          config = faithfulness_config,
      ),
    ]
)

The above example will lead to generation of new column

• FDL Faithfulness: Binary metric, which is True if the facts used inresponse is correctly used from the context columns.

Coherence (beta)

The Coherence enrichment assesses the logical flow and clarity of AI-generated text responses, ensuring they are structured in a way that makes sense from start to finish. This enrichment is crucial for evaluating whether the content produced by AI maintains a consistent theme, argument, or narrative, without disjointed thoughts or abrupt shifts in topic. Coherence is key to making AI-generated content not only understandable but also engaging and informative for the reader.

In applications ranging from storytelling and article generation to customer service interactions, coherence determines the effectiveness of communication. A coherent response builds trust in the AI's capabilities, as it demonstrates an understanding of not just language, but also context and the natural progression of ideas. This enrichment encourages AI systems to produce content that flows naturally, mimicking the way a knowledgeable human would convey information or tell a story.

For developers, integrating the Coherence enrichment into AI evaluation processes is essential for achieving outputs that resonate with human readers. It helps in fine-tuning AI models to produce content that not only answers questions or provides information but does so in a way that is logical and easy to follow. By prioritizing coherence, AI-generated texts can better serve their intended purpose, whether to inform, persuade, or entertain, enhancing the overall quality and impact of AI communications.

• To enable set enrichment parameter tocoherence.

Requirements:

• This enrichment requires access to the OpenAI API, which may introduce latency due to network communication and processing time. Learn more about LLM based enrichments
• OpenAI API access token MUST BE provided by the user.

coherence_config = {
  'response' : 'response_col',
}

fdl.ModelInfo.from_dataset_info(
    dataset_info=dataset_info,
    display_name='llm_model',
    model_task=fdl.core_objects.ModelTask.LLM,
    custom_features = [
      fdl.Enrichment(
          name = 'Coherence',
          enrichment = 'coherence', 
          columns = ['response_col'],
          config = coherence_config,
      ),
    ]
)

The above example will lead to generation of new column

• FDL Coherence: Binary metric, which is True ifresponse makes coherent arguments which flow well.

Conciseness (beta)

The Conciseness enrichment evaluates the brevity and clarity of AI-generated text responses, ensuring that the information is presented in a straightforward and efficient manner. This enrichment identifies and rewards responses that effectively communicate their message without unnecessary elaboration or redundancy. In the realm of AI-generated content, where verbosity can dilute the message's impact or confuse the audience, maintaining conciseness is crucial for enhancing readability and user engagement.

Implementing the Conciseness metric can significantly improve the user experience across various applications, from chatbots and virtual assistants to summarization tools and content generation platforms. It encourages the AI to distill information down to its essence, providing users with clear, to-the-point answers that satisfy their queries or needs without overwhelming them with superfluous details.

For developers and content creators, the Conciseness serves as a valuable tool for refining the output of AI systems, aligning them more closely with human preferences for communication that is both efficient and effective. By prioritizing conciseness, AI-generated content can become more accessible and useful, meeting the high standards of users who value quick and accurate information delivery. This enrichment, therefore, plays a pivotal role in the ongoing effort to enhance the quality and utility of AI-generated text, making it an indispensable component of AI evaluation frameworks.

• To enable set enrichment parameter toconciseness.

Requirements:

• This enrichment requires access to the OpenAI API, which may introduce latency due to network communication and processing time. Learn more about LLM based enrichments
• OpenAI API access token MUST BE provided by the user.

conciseness_config = {
  'response' : 'response_col',
}

fdl.ModelInfo.from_dataset_info(
    dataset_info=dataset_info,
    display_name='llm_model',
    model_task=fdl.core_objects.ModelTask.LLM,
    custom_features = [
      fdl.Enrichment(
          name = 'Conciseness',
          enrichment = 'conciseness', 
          columns = ['response_col'],
          config = coherence_config,
      ),
    ]
)

The above example will lead to generation of new column

• FDL Conciseness: Binary metric, which is True ifresponse is concise, and not overly verbose.

Toxicity (beta)

The toxicity enrichment classifies whether a piece of text is toxic or not. A RoBERTa based model is fine-tuned with a mix of toxic and non-toxic data. The model predicts score between 0-1 where scores closer to 1 indicate toxicity.

Following table provides the model performance on Toxic-Chat dataset, a real-world dataset that contains user prompt and responses in a chatbot setting.

Usage

The code snippet shows how to enable toxicity scoring on the prompt and response columns for each event published to Fiddler.

fdl.ModelInfo.from_dataset_info(
    dataset_info=dataset_info,
    display_name='llm_model',
    model_task=fdl.core_objects.ModelTask.LLM,
    custom_features = [
      fdl.Enrichment(
            name='Toxicity',
            enrichment='toxicity',
            columns=['prompt', 'response'],
        ),
    ]
)

The above example leads to creation of two columns each for prompt and response that contain the prediction probability and the model decision.

For example for the prompt column following two columns will be generated

• FDL Toxicity (prompt) toxicity_prob (float): model prediction probability between 0-1
• FDL Toxicity (prompt) contains_toxicity (bool): model prediction either 0 or 1

Regex Match (beta)

The Regex Match enrichment is designed to evaluate text responses or content based on their adherence to specific patterns defined by regular expressions (regex). By accepting a regex as input, this metric offers a highly customizable way to check if a string column in the dataset matches the given pattern. This functionality is essential for scenarios requiring precise formatting, specific keyword inclusion, or adherence to particular linguistic structures.

In practical applications, the Regex Match enrichment can be instrumental in validating data entries, ensuring compliance with formatting standards, or verifying the presence of required terms or codes within AI-generated content. Whether it's checking for email addresses, phone numbers, specific terminologies, or coding patterns, this metric provides a straightforward and efficient method for assessing the conformance of text to predefined patterns.

For developers and data analysts, the Regex Match enrichment is a powerful tool for automating the quality control process of textual data. It enables the swift identification of entries that fail to meet the necessary criteria, thereby streamlining the process of refining and improving the dataset or AI-generated content. This enrichment not only saves time but also enhances the reliability of data-driven applications by ensuring that the text outputs adhere closely to the desired specifications or standards.

Implementing the Regex Match enrichment into the evaluation and production framework of AI systems allows for a level of precision in text analysis that is crucial for applications demanding high accuracy and specificity. This metric is invaluable for maintaining the integrity and usefulness of textual data, making it a key component in the toolkit of anyone working with AI-generated content or large text datasets.

• To enable set enrichment parameter toregex_match.

fdl.ModelInfo.from_dataset_info(
    dataset_info=dataset_info,
    display_name='llm_model',
    model_task=fdl.core_objects.ModelTask.LLM,
    	custom_features = [
      	fdl.Enrichment(
	        name='Regex - only digits',
  	      enrichment='topic_model',
    	    columns=['question', 'response'],
      	  config = {
        	    'regex' : '^\d+$',
	        }
  	  	),
    ]
)

The above example will lead to generation of new column

• FDL Regex - only digits(category) : Match or No Match, depending on the regex specified in config matching in the string.

Topic (beta)

The Topic enrichment leverages the capabilities of Zero Shot Classifier Zero Shot Classifier models to categorize textual inputs into a predefined list of topics, even without having been explicitly trained on those topics. This approach to text classification is known as zero-shot learning, a groundbreaking method in natural language processing (NLP) that allows models to intelligently classify text into categories they haven't encountered during training. It's particularly useful for applications requiring the ability to understand and organize content dynamically across a broad range of subjects or themes.

By utilizing zero-shot classification, the Topic enrichment provides a flexible and powerful tool for automatically sorting and labeling text according to relevant topics. This is invaluable for content management systems, recommendation engines, and any application needing to quickly and accurately understand the thematic content of large volumes of text.

The enrichment works by evaluating the semantic similarity between the textual input and potential topic labels, assigning the most appropriate topic based on the content. This process enables the handling of diverse and evolving content types without the need for continual retraining or manual classification, significantly reducing the effort and complexity involved in content categorization.

Implementing the Topic enrichment into your data processing workflow can dramatically enhance the organization and accessibility of textual content, making it easier to deliver relevant, targeted information to users or to analyze content themes at scale. This enrichment taps into the advanced capabilities of zero-shot classification to provide a nuanced, efficient, and adaptable tool for text categorization, essential for anyone working with diverse and dynamic textual datasets.

Requirements

• Access to Huggingface inference endpoint - https://api-inference.huggingface.cowith model 'MoritzLaurer/mDeBERTa-v3-base-xnli-multilingual-nli-2mil7'
• Huggingface API token

fdl.ModelInfo.from_dataset_info(
    dataset_info=dataset_info,
    display_name='llm_model',
    model_task=fdl.core_objects.ModelTask.LLM,
    custom_features = [
      fdl.Enrichment(
          name='Topics',
          enrichment='topic_model',
          columns=['response'],
	        config={'topics':['politics', 'economy', 'astronomy']},        
      ),
    ]
)

The above example leads to creation of two columns -

• FDL Topics (response) topic_model_scores(List of float): List of floats indicating probability of the given column in each of the topics specified in the Enrichment config. Each float value indicate probability of the given input classified in the corresponding topic, in the same order as topics. Each value will be between 0 and 1. The sum of values does not equal to 1, as each classification is performed independently of other topics.
• FDL Topics (response) max_score_topic(string): Topic with the maximum score from the list of topic names specified in the Enrichment config.

Banned Keyword Detector (beta)

The Banned Keyword Detector enrichment is designed to scrutinize textual inputs for the presence of specified terms, particularly focusing on identifying content that includes potentially undesirable or restricted keywords. This enrichment operates based on a list of terms defined in its configuration, making it highly adaptable to various content moderation, compliance, and content filtering needs.

By specifying a list of terms to be flagged, the Banned Keyword Detector provides a straightforward yet powerful mechanism for automatically scanning and flagging content that contains certain keywords. This capability is crucial for platforms seeking to maintain high standards of content quality, adhere to regulatory requirements, or ensure community guidelines are followed. It's particularly valuable in environments where content is user-generated, providing

• To enable, set enrichment parameter tobanned_keywordsand specify a list of terms in the banned_keywords config parameter.

fdl.ModelInfo.from_dataset_info(
    dataset_info=dataset_info,
    display_name='llm_model',
    model_task=fdl.core_objects.ModelTask.LLM,
    custom_features = [
      fdl.Enrichment(
            name='Banned KW',
            enrichment='banned_keywords',
            columns=['prompt', 'response'],
            config={'output_column_name': 'contains_banned_kw', 'banned_keywords':['nike', 'adidas', 'puma'],},
        ),
    ]
)

The above example leads to creation of two columns -

• FDL Banned KW (prompt) contains_banned_kw(bool): to indicate if input contains one of the specified banned keywords in the value of the prompt column
• FDL Banned KW (response) contains_banned_kw(bool): to indicate if input contains one of the specified banned keywords in the value of the response column

fdl.BaselineType

from fiddler import BaselineType

PROJECT_NAME = 'example_project'
BASELINE_NAME = 'example_rolling'
DATASET_NAME = 'example_validation'
MODEL_NAME = 'example_model'

client.add_baseline(
  project_id=PROJECT_NAME,
  model_id=MODEL_NAME,
  baseline_id=BASELINE_NAME,
  type=BaselineType.PRE_PRODUCTION,
  dataset_id=DATASET_NAME,
)

DataSource

fdl.DatasetDataSource

DATASET_ID = 'example_dataset'

data_source = fdl.DatasetDataSource(
    dataset_id=DATASET_ID
  	source='baseline.csv',
  	num_samples=500,
)

fdl.SqlSliceQueryDataSource

DATASET_ID = 'example_dataset'
MODEL_ID = 'example_model'

query = f'SELECT * FROM {DATASET_ID}.{MODEL_ID} WHERE CreditScore > 700'
data_source = fdl.SqlSliceQueryDataSource(
    query=query,
  	num_samples=500,
)

fdl.RowDataSource

row = df.to_dict(orient='records')[0]

data_source = fdl.RowDataSource(
    row=row,
)

fdl.EventIdDataSource

DATASET_ID = 'example_dataset'

# In Dataset table
data_source = fdl.EventIdDataSource(
    event_id='xGhys7-83HgdtsoiuYTa872',
  	dataset_name=DATASET_ID,
)

# In Production table
data_source = fdl.EventIdDataSource(
    event_id='xGhys7-83HgdtsoiuYTa872',
  	dataset_name='production',
)

fdl.Webhook

webhook = fdl.Webhook(
    name='data_integrity_violations_channel',
    url='https://hooks.slack.com/services/T9EAVLUQ5/P982J/G8ISUczk37hxQ15C28d',
    provider='SLACK',
  	uuid='74a4fdcf-34eb-4dc3-9a79-e48e14cca686',
    organization_name='some_org',
)

Example Response:

Webhook(name='data_integrity_violations_channel',
    url='https://hooks.slack.com/services/T9EAVLUQ5/P982J/G8ISUczk37hxQ15C28d',
    provider='SLACK',
  	uuid='74a4fdcf-34eb-4dc3-9a79-e48e14cca686',
    organization_name='some_org',
)