Aggregate Agent Observability experimentation

v2 (latest)

Note: This endpoint is in preview and is subject to change. If you have any feedback, contact Datadog support.

POST https://api.ap1.datadoghq.com/api/v2/llm-obs/v1/experimentation/analyticshttps://api.ap2.datadoghq.com/api/v2/llm-obs/v1/experimentation/analyticshttps://api.datadoghq.eu/api/v2/llm-obs/v1/experimentation/analyticshttps://api.ddog-gov.com/api/v2/llm-obs/v1/experimentation/analyticshttps://api.us2.ddog-gov.com/api/v2/llm-obs/v1/experimentation/analyticshttps://api.uk1.datadoghq.com/api/v2/llm-obs/v1/experimentation/analyticshttps://api.datadoghq.com/api/v2/llm-obs/v1/experimentation/analyticshttps://api.us3.datadoghq.com/api/v2/llm-obs/v1/experimentation/analyticshttps://api.us5.datadoghq.com/api/v2/llm-obs/v1/experimentation/analytics

Overview

Execute an analytics aggregation over Agent Observability experimentation data. Use this endpoint to compute metrics (for example average eval scores) grouped by fields such as span_id or experiment_id.

At least one compute definition and one index must be provided.

Request

Body Data (required)

Analytics payload.

Expand All

Field

Type

Description

data [required]

object

Data object for an analytics request.

{
  "data": {
    "attributes": {
      "aggregate": {
        "compute": [
          {
            "metric": "score_value",
            "name": "avg_faithfulness"
          }
        ],
        "dataset_version": "integer",
        "group_by": [
          {
            "field": "span_id"
          }
        ],
        "indexes": [
          "experiment-evals"
        ],
        "limit": 1000,
        "search": {
          "query": "@experiment_id:3fd6b5e0-8910-4b1c-a7d0-5b84de329012"
        },
        "time": {
          "from": 1705312200000,
          "to": 1705315800000
        }
      }
    },
    "type": "experimentation"
  }
}

Response

Response to an analytics query.

Expand All

Field

Type

Description

data [required]

object

JSON:API data object for an analytics response.

{
  "data": {
    "attributes": {
      "hit_count": 1500,
      "result": {
        "values": [
          {
            "by": {
              "span_id": "span-7a1b2c3d"
            },
            "metrics": {
              "score_value": 0.85
            }
          }
        ]
      }
    },
    "id": "00000000-0000-0000-0000-000000000001",
    "type": "experimentation"
  }
}

Bad Request

API error response.

Expand All

Field

Type

Description

errors [required]

[object]

A list of errors.

{
  "errors": [
    {
      "detail": "Missing required attribute in body",
      "meta": {},
      "source": {
        "header": "Authorization",
        "parameter": "limit",
        "pointer": "/data/attributes/title"
      },
      "status": "400",
      "title": "Bad Request"
    }
  ]
}

Unauthorized

API error response.

Expand All

Field

Type

Description

errors [required]

[object]

A list of errors.

{
  "errors": [
    {
      "detail": "Missing required attribute in body",
      "meta": {},
      "source": {
        "header": "Authorization",
        "parameter": "limit",
        "pointer": "/data/attributes/title"
      },
      "status": "400",
      "title": "Bad Request"
    }
  ]
}

Forbidden

API error response.

Expand All

Field

Type

Description

errors [required]

[object]

A list of errors.

{
  "errors": [
    {
      "detail": "Missing required attribute in body",
      "meta": {},
      "source": {
        "header": "Authorization",
        "parameter": "limit",
        "pointer": "/data/attributes/title"
      },
      "status": "400",
      "title": "Bad Request"
    }
  ]
}

Too many requests

API error response.

Expand All

Field

Type

Description

errors [required]

[string]

A list of errors.

{
  "errors": [
    "Bad Request"
  ]
}

Internal Server Error

API error response.

Expand All

Field

Type

Description

errors [required]

[object]

A list of errors.

{
  "errors": [
    {
      "detail": "Missing required attribute in body",
      "meta": {},
      "source": {
        "header": "Authorization",
        "parameter": "limit",
        "pointer": "/data/attributes/title"
      },
      "status": "400",
      "title": "Bad Request"
    }
  ]
}

Code Example

                  ## default
# 

# Curl command
curl -X POST "https://api.ap1.datadoghq.com"https://api.ap2.datadoghq.com"https://api.datadoghq.eu"https://api.ddog-gov.com"https://api.us2.ddog-gov.com"https://api.uk1.datadoghq.com"https://api.datadoghq.com"https://api.us3.datadoghq.com"https://api.us5.datadoghq.com/api/v2/llm-obs/v1/experimentation/analytics" \
-H "Accept: application/json" \
-H "Content-Type: application/json" \
-H "DD-API-KEY: ${DD_API_KEY}" \
-H "DD-APPLICATION-KEY: ${DD_APP_KEY}" \
-d @- << EOF
{
  "data": {
    "attributes": {
      "aggregate": {
        "compute": [
          {
            "metric": "score_value",
            "name": "avg_faithfulness"
          }
        ],
        "group_by": [
          {
            "field": "span_id"
          }
        ],
        "indexes": [
          "experiment-evals"
        ],
        "search": {
          "query": "@experiment_id:3fd6b5e0-8910-4b1c-a7d0-5b84de329012 @label:faithfulness"
        }
      }
    },
    "type": "experimentation"
  }
}
EOF

                

"""
Aggregate Agent Observability experimentation returns "OK" response
"""

from datadog_api_client import ApiClient, Configuration
from datadog_api_client.v2.api.agent_observability_api import AgentObservabilityApi
from datadog_api_client.v2.model.llm_obs_experimentation_analytics_aggregate import (
    LLMObsExperimentationAnalyticsAggregate,
)
from datadog_api_client.v2.model.llm_obs_experimentation_analytics_compute import LLMObsExperimentationAnalyticsCompute
from datadog_api_client.v2.model.llm_obs_experimentation_analytics_data_attributes_request import (
    LLMObsExperimentationAnalyticsDataAttributesRequest,
)
from datadog_api_client.v2.model.llm_obs_experimentation_analytics_data_request import (
    LLMObsExperimentationAnalyticsDataRequest,
)
from datadog_api_client.v2.model.llm_obs_experimentation_analytics_group_by import LLMObsExperimentationAnalyticsGroupBy
from datadog_api_client.v2.model.llm_obs_experimentation_analytics_request import LLMObsExperimentationAnalyticsRequest
from datadog_api_client.v2.model.llm_obs_experimentation_analytics_search import LLMObsExperimentationAnalyticsSearch
from datadog_api_client.v2.model.llm_obs_experimentation_analytics_time_range import (
    LLMObsExperimentationAnalyticsTimeRange,
)
from datadog_api_client.v2.model.llm_obs_experimentation_type import LLMObsExperimentationType

body = LLMObsExperimentationAnalyticsRequest(
    data=LLMObsExperimentationAnalyticsDataRequest(
        attributes=LLMObsExperimentationAnalyticsDataAttributesRequest(
            aggregate=LLMObsExperimentationAnalyticsAggregate(
                compute=[
                    LLMObsExperimentationAnalyticsCompute(
                        metric="score_value",
                        name="avg_faithfulness",
                    ),
                ],
                dataset_version=None,
                group_by=[
                    LLMObsExperimentationAnalyticsGroupBy(
                        field="span_id",
                    ),
                ],
                indexes=[
                    "experiment-evals",
                ],
                limit=1000,
                search=LLMObsExperimentationAnalyticsSearch(
                    query="@experiment_id:3fd6b5e0-8910-4b1c-a7d0-5b84de329012",
                ),
                time=LLMObsExperimentationAnalyticsTimeRange(
                    _from=1705312200000,
                    to=1705315800000,
                ),
            ),
        ),
        type=LLMObsExperimentationType.EXPERIMENTATION,
    ),
)

configuration = Configuration()
configuration.unstable_operations["aggregate_llm_obs_experimentation"] = True
with ApiClient(configuration) as api_client:
    api_instance = AgentObservabilityApi(api_client)
    response = api_instance.aggregate_llm_obs_experimentation(body=body)

    print(response)

Instructions

First install the library and its dependencies and then save the example to example.py and run following commands:

    DD_SITE="datadoghq.comus3.datadoghq.comus5.datadoghq.comdatadoghq.euap1.datadoghq.comap2.datadoghq.comuk1.datadoghq.comddog-gov.comus2.ddog-gov.com" DD_API_KEY="<DD_API_KEY>" DD_APP_KEY="<DD_APP_KEY>" python3 "example.py"

# Aggregate Agent Observability experimentation returns "OK" response

require "datadog_api_client"
DatadogAPIClient.configure do |config|
  config.unstable_operations["v2.aggregate_llm_obs_experimentation".to_sym] = true
end
api_instance = DatadogAPIClient::V2::AgentObservabilityAPI.new

body = DatadogAPIClient::V2::LLMObsExperimentationAnalyticsRequest.new({
  data: DatadogAPIClient::V2::LLMObsExperimentationAnalyticsDataRequest.new({
    attributes: DatadogAPIClient::V2::LLMObsExperimentationAnalyticsDataAttributesRequest.new({
      aggregate: DatadogAPIClient::V2::LLMObsExperimentationAnalyticsAggregate.new({
        compute: [
          DatadogAPIClient::V2::LLMObsExperimentationAnalyticsCompute.new({
            metric: "score_value",
            name: "avg_faithfulness",
          }),
        ],
        dataset_version: nil,
        group_by: [
          DatadogAPIClient::V2::LLMObsExperimentationAnalyticsGroupBy.new({
            field: "span_id",
          }),
        ],
        indexes: [
          "experiment-evals",
        ],
        limit: 1000,
        search: DatadogAPIClient::V2::LLMObsExperimentationAnalyticsSearch.new({
          query: "@experiment_id:3fd6b5e0-8910-4b1c-a7d0-5b84de329012",
        }),
        time: DatadogAPIClient::V2::LLMObsExperimentationAnalyticsTimeRange.new({
          from: 1705312200000,
          to: 1705315800000,
        }),
      }),
    }),
    type: DatadogAPIClient::V2::LLMObsExperimentationType::EXPERIMENTATION,
  }),
})
p api_instance.aggregate_llm_obs_experimentation(body)

Instructions

First install the library and its dependencies and then save the example to example.rb and run following commands:

    DD_SITE="datadoghq.comus3.datadoghq.comus5.datadoghq.comdatadoghq.euap1.datadoghq.comap2.datadoghq.comuk1.datadoghq.comddog-gov.comus2.ddog-gov.com" DD_API_KEY="<DD_API_KEY>" DD_APP_KEY="<DD_APP_KEY>" rb "example.rb"

// Aggregate Agent Observability experimentation returns "OK" response

package main

import (
	"context"
	"encoding/json"
	"fmt"
	"os"

	"github.com/DataDog/datadog-api-client-go/v2/api/datadog"
	"github.com/DataDog/datadog-api-client-go/v2/api/datadogV2"
)

func main() {
	body := datadogV2.LLMObsExperimentationAnalyticsRequest{
		Data: datadogV2.LLMObsExperimentationAnalyticsDataRequest{
			Attributes: datadogV2.LLMObsExperimentationAnalyticsDataAttributesRequest{
				Aggregate: datadogV2.LLMObsExperimentationAnalyticsAggregate{
					Compute: []datadogV2.LLMObsExperimentationAnalyticsCompute{
						{
							Metric: "score_value",
							Name:   datadog.PtrString("avg_faithfulness"),
						},
					},
					DatasetVersion: *datadog.NewNullableInt64(nil),
					GroupBy: []datadogV2.LLMObsExperimentationAnalyticsGroupBy{
						{
							Field: "span_id",
						},
					},
					Indexes: []string{
						"experiment-evals",
					},
					Limit: *datadog.NewNullableInt32(datadog.PtrInt32(1000)),
					Search: datadogV2.LLMObsExperimentationAnalyticsSearch{
						Query: "@experiment_id:3fd6b5e0-8910-4b1c-a7d0-5b84de329012",
					},
					Time: &datadogV2.LLMObsExperimentationAnalyticsTimeRange{
						From: 1705312200000,
						To:   1705315800000,
					},
				},
			},
			Type: datadogV2.LLMOBSEXPERIMENTATIONTYPE_EXPERIMENTATION,
		},
	}
	ctx := datadog.NewDefaultContext(context.Background())
	configuration := datadog.NewConfiguration()
	configuration.SetUnstableOperationEnabled("v2.AggregateLLMObsExperimentation", true)
	apiClient := datadog.NewAPIClient(configuration)
	api := datadogV2.NewAgentObservabilityApi(apiClient)
	resp, r, err := api.AggregateLLMObsExperimentation(ctx, body)

	if err != nil {
		fmt.Fprintf(os.Stderr, "Error when calling `AgentObservabilityApi.AggregateLLMObsExperimentation`: %v\n", err)
		fmt.Fprintf(os.Stderr, "Full HTTP response: %v\n", r)
	}

	responseContent, _ := json.MarshalIndent(resp, "", "  ")
	fmt.Fprintf(os.Stdout, "Response from `AgentObservabilityApi.AggregateLLMObsExperimentation`:\n%s\n", responseContent)
}

Instructions

First install the library and its dependencies and then save the example to main.go and run following commands:

    DD_SITE="datadoghq.comus3.datadoghq.comus5.datadoghq.comdatadoghq.euap1.datadoghq.comap2.datadoghq.comuk1.datadoghq.comddog-gov.comus2.ddog-gov.com" DD_API_KEY="<DD_API_KEY>" DD_APP_KEY="<DD_APP_KEY>" go run "main.go"

// Aggregate Agent Observability experimentation returns "OK" response

import com.datadog.api.client.ApiClient;
import com.datadog.api.client.ApiException;
import com.datadog.api.client.v2.api.AgentObservabilityApi;
import com.datadog.api.client.v2.model.LLMObsExperimentationAnalyticsAggregate;
import com.datadog.api.client.v2.model.LLMObsExperimentationAnalyticsCompute;
import com.datadog.api.client.v2.model.LLMObsExperimentationAnalyticsDataAttributesRequest;
import com.datadog.api.client.v2.model.LLMObsExperimentationAnalyticsDataRequest;
import com.datadog.api.client.v2.model.LLMObsExperimentationAnalyticsGroupBy;
import com.datadog.api.client.v2.model.LLMObsExperimentationAnalyticsRequest;
import com.datadog.api.client.v2.model.LLMObsExperimentationAnalyticsResponse;
import com.datadog.api.client.v2.model.LLMObsExperimentationAnalyticsSearch;
import com.datadog.api.client.v2.model.LLMObsExperimentationAnalyticsTimeRange;
import com.datadog.api.client.v2.model.LLMObsExperimentationType;
import java.util.Collections;

public class Example {
  public static void main(String[] args) {
    ApiClient defaultClient = ApiClient.getDefaultApiClient();
    defaultClient.setUnstableOperationEnabled("v2.aggregateLLMObsExperimentation", true);
    AgentObservabilityApi apiInstance = new AgentObservabilityApi(defaultClient);

    LLMObsExperimentationAnalyticsRequest body =
        new LLMObsExperimentationAnalyticsRequest()
            .data(
                new LLMObsExperimentationAnalyticsDataRequest()
                    .attributes(
                        new LLMObsExperimentationAnalyticsDataAttributesRequest()
                            .aggregate(
                                new LLMObsExperimentationAnalyticsAggregate()
                                    .compute(
                                        Collections.singletonList(
                                            new LLMObsExperimentationAnalyticsCompute()
                                                .metric("score_value")
                                                .name("avg_faithfulness")))
                                    .datasetVersion(null)
                                    .groupBy(
                                        Collections.singletonList(
                                            new LLMObsExperimentationAnalyticsGroupBy()
                                                .field("span_id")))
                                    .indexes(Collections.singletonList("experiment-evals"))
                                    .limit(1000)
                                    .search(
                                        new LLMObsExperimentationAnalyticsSearch()
                                            .query(
                                                "@experiment_id:3fd6b5e0-8910-4b1c-a7d0-5b84de329012"))
                                    .time(
                                        new LLMObsExperimentationAnalyticsTimeRange()
                                            .from(1705312200000L)
                                            .to(1705315800000L))))
                    .type(LLMObsExperimentationType.EXPERIMENTATION));

    try {
      LLMObsExperimentationAnalyticsResponse result =
          apiInstance.aggregateLLMObsExperimentation(body);
      System.out.println(result);
    } catch (ApiException e) {
      System.err.println(
          "Exception when calling AgentObservabilityApi#aggregateLLMObsExperimentation");
      System.err.println("Status code: " + e.getCode());
      System.err.println("Reason: " + e.getResponseBody());
      System.err.println("Response headers: " + e.getResponseHeaders());
      e.printStackTrace();
    }
  }
}

Instructions

First install the library and its dependencies and then save the example to Example.java and run following commands:

    DD_SITE="datadoghq.comus3.datadoghq.comus5.datadoghq.comdatadoghq.euap1.datadoghq.comap2.datadoghq.comuk1.datadoghq.comddog-gov.comus2.ddog-gov.com" DD_API_KEY="<DD_API_KEY>" DD_APP_KEY="<DD_APP_KEY>" java "Example.java"

// Aggregate Agent Observability experimentation returns "OK" response
use datadog_api_client::datadog;
use datadog_api_client::datadogV2::api_agent_observability::AgentObservabilityAPI;
use datadog_api_client::datadogV2::model::LLMObsExperimentationAnalyticsAggregate;
use datadog_api_client::datadogV2::model::LLMObsExperimentationAnalyticsCompute;
use datadog_api_client::datadogV2::model::LLMObsExperimentationAnalyticsDataAttributesRequest;
use datadog_api_client::datadogV2::model::LLMObsExperimentationAnalyticsDataRequest;
use datadog_api_client::datadogV2::model::LLMObsExperimentationAnalyticsGroupBy;
use datadog_api_client::datadogV2::model::LLMObsExperimentationAnalyticsRequest;
use datadog_api_client::datadogV2::model::LLMObsExperimentationAnalyticsSearch;
use datadog_api_client::datadogV2::model::LLMObsExperimentationAnalyticsTimeRange;
use datadog_api_client::datadogV2::model::LLMObsExperimentationType;

#[tokio::main]
async fn main() {
    let body =
        LLMObsExperimentationAnalyticsRequest::new(LLMObsExperimentationAnalyticsDataRequest::new(
            LLMObsExperimentationAnalyticsDataAttributesRequest::new(
                LLMObsExperimentationAnalyticsAggregate::new(
                    vec![
                        LLMObsExperimentationAnalyticsCompute::new("score_value".to_string())
                            .name("avg_faithfulness".to_string()),
                    ],
                    vec!["experiment-evals".to_string()],
                    LLMObsExperimentationAnalyticsSearch::new(
                        "@experiment_id:3fd6b5e0-8910-4b1c-a7d0-5b84de329012".to_string(),
                    ),
                )
                .dataset_version(None)
                .group_by(vec![LLMObsExperimentationAnalyticsGroupBy::new(
                    "span_id".to_string(),
                )])
                .limit(Some(1000))
                .time(LLMObsExperimentationAnalyticsTimeRange::new(
                    1705312200000,
                    1705315800000,
                )),
            ),
            LLMObsExperimentationType::EXPERIMENTATION,
        ));
    let mut configuration = datadog::Configuration::new();
    configuration.set_unstable_operation_enabled("v2.AggregateLLMObsExperimentation", true);
    let api = AgentObservabilityAPI::with_config(configuration);
    let resp = api.aggregate_llm_obs_experimentation(body).await;
    if let Ok(value) = resp {
        println!("{:#?}", value);
    } else {
        println!("{:#?}", resp.unwrap_err());
    }
}

Instructions

First install the library and its dependencies and then save the example to src/main.rs and run following commands:

    DD_SITE="datadoghq.comus3.datadoghq.comus5.datadoghq.comdatadoghq.euap1.datadoghq.comap2.datadoghq.comuk1.datadoghq.comddog-gov.comus2.ddog-gov.com" DD_API_KEY="<DD_API_KEY>" DD_APP_KEY="<DD_APP_KEY>" cargo run

/**
 * Aggregate Agent Observability experimentation returns "OK" response
 */

import { client, v2 } from "@datadog/datadog-api-client";

const configuration = client.createConfiguration();
configuration.unstableOperations["v2.aggregateLLMObsExperimentation"] = true;
const apiInstance = new v2.AgentObservabilityApi(configuration);

const params: v2.AgentObservabilityApiAggregateLLMObsExperimentationRequest = {
  body: {
    data: {
      attributes: {
        aggregate: {
          compute: [
            {
              metric: "score_value",
              name: "avg_faithfulness",
            },
          ],
          datasetVersion: undefined,
          groupBy: [
            {
              field: "span_id",
            },
          ],
          indexes: ["experiment-evals"],
          limit: 1000,
          search: {
            query: "@experiment_id:3fd6b5e0-8910-4b1c-a7d0-5b84de329012",
          },
          time: {
            from: 1705312200000,
            to: 1705315800000,
          },
        },
      },
      type: "experimentation",
    },
  },
};

apiInstance
  .aggregateLLMObsExperimentation(params)
  .then((data: v2.LLMObsExperimentationAnalyticsResponse) => {
    console.log(
      "API called successfully. Returned data: " + JSON.stringify(data)
    );
  })
  .catch((error: any) => console.error(error));

Instructions

First install the library and its dependencies and then save the example to example.ts and run following commands:

    DD_SITE="datadoghq.comus3.datadoghq.comus5.datadoghq.comdatadoghq.euap1.datadoghq.comap2.datadoghq.comuk1.datadoghq.comddog-gov.comus2.ddog-gov.com" DD_API_KEY="<DD_API_KEY>" DD_APP_KEY="<DD_APP_KEY>" tsc "example.ts"

Aggregate Agent Observability experimentation

Overview

Request

Body Data (required)

Response

Code Example

Aggregate Agent Observability experimentation

Aggregate Agent Observability experimentation

Instructions

Aggregate Agent Observability experimentation

Instructions

Aggregate Agent Observability experimentation

Instructions

Aggregate Agent Observability experimentation

Instructions

Aggregate Agent Observability experimentation

Instructions

Aggregate Agent Observability experimentation

Instructions

How can I help you today?