Kong AI Gateway Integration

_format_version: "3.0"

services:
  - name: openai-service
    host: api.openai.com
    port: 443
    protocol: https
    routes:
      - name: openai-chat-route
        paths:
          - /openai
        strip_path: true

plugins:
  - name: ai-proxy
    service: openai-service
    config:
      route_type: llm/v1/chat
      auth:
        header_name: Authorization
        header_value: "Bearer ${OPENAI_API_KEY}"
      model:
        provider: openai
        name: gpt-4o-mini
        options:
          max_tokens: 512
          temperature: 0.7
      # Set log_payloads: true to include prompt and completion text in OTel spans.
      # WARNING: payloads may contain PII. Disable in production unless needed.
      logging:
        log_statistics: true
        log_payloads: true

  # Session grouping: read the X-Fiddler-Conversation-Id request header and stamp
  # it as gen_ai.conversation.id on the request's OTel spans, so multi-turn calls
  # sharing one conversation id group into a single Fiddler Session.
  # See the "Session Grouping" section below for how this works.
  - name: pre-function
    config:
      access:
        - |
          local conv_id = kong.request.get_header("X-Fiddler-Conversation-Id")
          if conv_id and conv_id ~= "" then
            ngx.ctx.fiddler_conversation_id = conv_id
          end
      header_filter:
        - |
          local conv_id = ngx.ctx.fiddler_conversation_id
          if conv_id then
            local spans = ngx.ctx.KONG_SPANS
            if spans then
              for _, span in ipairs(spans) do
                span:set_attribute("gen_ai.conversation.id", conv_id)
              end
            end
          end

  - name: opentelemetry
    config:
      traces_endpoint: "${FIDDLER_URL}/v1/traces"
      headers:
        Authorization: "Bearer ${FIDDLER_API_KEY}"
        fiddler-application-id: "${FIDDLER_APP_ID}"
      resource_attributes:
        service.name: kong
        application.id: "${FIDDLER_APP_ID}"
      propagation:
        default_format: w3c

import os
import uuid
from openai import OpenAI

KONG_URL = os.getenv("KONG_URL", "http://localhost:8000")

# api_key is not forwarded to OpenAI — Kong handles auth via ai-proxy.
client = OpenAI(base_url=f"{KONG_URL}/openai", api_key="kong-managed")

# Generate one conversation UUID per session and send it on every LLM call
# so Fiddler groups them into a single Session.
session_id = str(uuid.uuid4())

response = client.chat.completions.create(
    model="gpt-4o-mini",
    messages=[{"role": "user", "content": "What is the capital of France?"}],
    extra_headers={"X-Fiddler-Conversation-Id": session_id},
)
print(response.choices[0].message.content)

export KONG_URL="http://my-kong-host:8000"

import uuid
conversation_id = str(uuid.uuid4())  # one per conversation

client.chat.completions.create(
    model="gpt-4o-mini",
    messages=[{"role": "user", "content": "..."}],
    extra_headers={"X-Fiddler-Conversation-Id": conversation_id},
)

grep '\${' kong_fiddler_config.yaml   # prints nothing when fully filled in