config object is used to configure API interactions with various providers. It supports multiple modes such as single provider access, load balancing between providers, and fallback strategies.
The following JSON schema is used to validate the config object:
Portkey Config JSON Schema
Portkey Config JSON Schema
{
"$schema": "http://json-schema.org/draft-07/schema#",
"type": "object",
"properties": {
"after_request_hooks": {
"type": "array",
"items": {
"properties": {
"id": {
"type": "string"
},
"type": {
"type": "string"
},
"async": {
"type": "boolean"
},
"on_fail": {
"type": "object",
"properties": {
"feedback": {
"type": "object",
"properties": {
"value": {
"type": "number"
},
"weight": {
"type": "number"
},
"metadata": {
"type": "object"
}
}
}
}
},
"on_success": {
"type": "object",
"properties": {
"feedback": {
"type": "object",
"properties": {
"value": {
"type": "number"
},
"weight": {
"type": "number"
},
"metadata": {
"type": "object"
}
}
}
}
},
"checks": {
"type": "array",
"items": {
"type": "object",
"properties": {
"id": {
"type": "string"
},
"parameters": {
"type": "object"
}
},
"required": ["id", "parameters"]
}
}
},
"required": ["id"]
}
},
"input_guardrails": {
"type": "array",
"items": {
"oneOf": [
{
"type": "object",
"properties": {
"id": {
"type": "string"
},
"deny": {
"type": "boolean"
},
"on_fail": {
"type": "object",
"properties": {
"feedback": {
"type": "object",
"properties": {
"value": {
"type": "number"
},
"weight": {
"type": "number"
},
"metadata": {
"type": "object"
}
}
}
}
},
"on_success": {
"type": "object",
"properties": {
"feedback": {
"type": "object",
"properties": {
"value": {
"type": "number"
},
"weight": {
"type": "number"
},
"metadata": {
"type": "object"
}
}
}
}
},
"async": {
"type": "boolean"
}
},
"additionalProperties": {
"type": "object",
"additionalProperties": true
}
},
{
"type": "string"
}
]
}
},
"output_guardrails": {
"type": "array",
"items": {
"oneOf": [
{
"type": "object",
"properties": {
"id": {
"type": "string"
},
"deny": {
"type": "boolean"
},
"on_fail": {
"type": "object",
"properties": {
"feedback": {
"type": "object",
"properties": {
"value": {
"type": "number"
},
"weight": {
"type": "number"
},
"metadata": {
"type": "object"
}
}
},
"deny": {
"type": "boolean"
}
}
},
"on_success": {
"type": "object",
"properties": {
"feedback": {
"type": "object",
"properties": {
"value": {
"type": "number"
},
"weight": {
"type": "number"
},
"metadata": {
"type": "object"
}
}
},
"deny": {
"type": "boolean"
}
}
},
"async": {
"type": "boolean"
}
},
"additionalProperties": {
"type": "object",
"additionalProperties": true
}
},
{
"type": "string"
}
]
}
},
"before_request_hooks": {
"type": "array",
"items": {
"properties": {
"id": {
"type": "string"
},
"type": {
"type": "string"
},
"on_fail": {
"type": "object",
"properties": {
"feedback": {
"type": "object",
"properties": {
"value": {
"type": "number"
},
"weight": {
"type": "number"
},
"metadata": {
"type": "object"
}
}
},
"deny": {
"type": "boolean"
}
}
},
"on_success": {
"type": "object",
"properties": {
"feedback": {
"type": "object",
"properties": {
"value": {
"type": "number"
},
"weight": {
"type": "number"
},
"metadata": {
"type": "object"
}
}
},
"deny": {
"type": "boolean"
}
}
},
"checks": {
"type": "array",
"items": {
"type": "object",
"properties": {
"id": {
"type": "string"
},
"parameters": {
"type": "object"
}
},
"required": ["id", "parameters"]
}
}
},
"required": ["id"]
}
},
"strategy": {
"type": "object",
"properties": {
"mode": {
"type": "string",
"enum": ["single", "loadbalance", "fallback", "conditional"]
},
"conditions": {
"type": "array",
"items": {
"type": "object",
"properties": {
"query": {
"type": "object"
},
"then": {
"type": "string"
}
},
"required": ["query", "then"]
}
},
"default": {
"type": "string"
},
"on_status_codes": {
"type": "array",
"items": {
"type": "integer"
},
"optional": true
}
},
"allOf": [
{
"if": {
"properties": {
"mode": {
"const": "conditional"
}
}
},
"then": {
"required": ["conditions", "default"]
}
}
],
"required": ["mode"]
},
"name": {
"type": "string"
},
"strict_open_ai_compliance": {
"type": "boolean"
},
"provider": {
"type": "string"
},
"resource_name": {
"type": "string",
"optional": true
},
"deployment_id": {
"type": "string",
"optional": true
},
"api_version": {
"type": "string",
"optional": true
},
"deployments": {
"type": "array",
"optional": true,
"items": {
"type": "object",
"properties": {
"deployment_id": {
"type": "string"
},
"alias": {
"type": "string"
},
"api_version": {
"type": "string"
},
"is_default": {
"type": "boolean"
}
},
"required": ["deployment_id", "alias", "api_version"]
}
},
"override_params": {
"type": "object"
},
"api_key": {
"type": "string"
},
"virtual_key": {
"type": "string"
},
"prompt_id": {
"type": "string"
},
"request_timeout": {
"type": "integer"
},
"cache": {
"type": "object",
"properties": {
"mode": {
"type": "string",
"enum": ["simple", "semantic"]
},
"max_age": {
"type": "integer",
"optional": true
}
},
"required": ["mode"]
},
"cb_config": {
"type": "object",
"properties": {
"failure_threshold": {
"type": "number",
"minimum": 1
},
"cooldown_interval": {
"type": "number",
"minimum": 30000
},
"failure_status_codes": {
"type": "array",
"items": {
"type": "integer"
},
"optional": true
}
},
"required": ["failure_threshold", "cooldown_interval"]
},
"retry": {
"type": "object",
"properties": {
"attempts": {
"type": "integer"
},
"use_retry_after_headers": {
"type": "boolean"
},
"on_status_codes": {
"type": "array",
"items": {
"type": "number"
},
"optional": true
}
},
"required": ["attempts"]
},
"weight": {
"type": "number"
},
"on_status_codes": {
"type": "array",
"items": {
"type": "integer"
}
},
"custom_host": {
"type": "string"
},
"forward_headers": {
"type": "array",
"items": {
"type": "string"
}
},
"targets": {
"type": "array",
"items": {
"$ref": "#"
}
},
"aws_access_key_id": {
"type": "string"
},
"aws_secret_access_key": {
"type": "string"
},
"aws_region": {
"type": "string"
},
"aws_session_token": {
"type": "string"
},
"openai_organization": {
"type": "string"
},
"openai_project": {
"type": "string"
},
"vertex_project_id": {
"type": "string"
},
"vertex_region": {
"type": "string"
},
"vertex_service_account_json": {
"type": "object"
},
"azure_region": {
"type": "string"
},
"azure_deployment_name": {
"type": "string"
},
"azure_deployment_type": {
"type": "string",
"enum": ["serverless", "managed"]
},
"azure_endpoint_name": {
"type": "string"
},
"azure_api_version": {
"type": "string"
}
},
"anyOf": [
{
"required": ["provider", "api_key"]
},
{
"required": ["provider", "custom_host"]
},
{
"required": ["virtual_key"]
},
{
"required": ["strategy", "targets"]
},
{
"required": ["cache"]
},
{
"required": ["retry"]
},
{
"required": ["prompt_id"]
},
{
"required": ["forward_headers"]
},
{
"required": ["request_timeout"]
},
{
"required": ["provider", "aws_access_key_id", "aws_secret_access_key"]
},
{
"required": ["provider", "vertex_region", "vertex_service_account_json"]
},
{
"required": ["provider", "vertex_region", "vertex_project_id"]
},
{
"required": [
"provider",
"azure_deployment_name",
"azure_deployment_type",
"azure_region",
"azure_api_version"
]
},
{
"required": ["provider", "azure_endpoint_name", "azure_deployment_type"]
},
{
"required": ["after_request_hooks"]
},
{
"required": ["before_request_hooks"]
},
{
"required": ["input_guardrails"]
},
{
"required": ["output_guardrails"]
}
],
"additionalProperties": false
}
Example Configs
// Using provider slug from Model Catalog (recommended)
{
"provider": "@openai-prod", // Provider slug from Model Catalog
"cache": { // Optional
"mode": "simple",
},
"retry": { // Optional
"attempts": 5,
"on_status_codes": []
}
}
// Using virtual_key (legacy, still supported)
{
"virtual_key": "your-virtual-key-slug",
"cache": { "mode": "semantic", "max_age": 10000 }
}
// Load balancing with provider slugs
{
"strategy": { "mode": "loadbalance" },
"targets": [
{ "provider": "@openai-prod" },
{ "provider": "@openai-backup" }
]
}
Schema Details
| Key Name | Description | Type | Required | Enum Values | Additional Info |
|---|---|---|---|---|---|
strategy | Operational strategy for the config or any individual target | object | Yes (if no provider or virtual_key) | - | See Strategy Object Details |
provider | Name of the service provider | string | Yes (if no mode or virtual_key) | @provider-slug OR “openai”, “anthropic”, “azure-openai”, “anyscale”, “cohere” | - |
api_key | API key for the service provider | string | Yes (if provider is specified) | - | - |
virtual_key | Virtual key identifier | string | Yes (if no mode or provider) | - | - |
cache | Caching configuration | object | No | - | See Cache Object Details |
retry | Retry configuration | object | No | - | See Retry Object Details |
weight | Weight for load balancing | number | No | - | Used in loadbalance mode |
on_status_codes | Status codes triggering fallback | array of strings | No | - | Used in fallback mode |
targets | List of target configurations | array | Yes (if mode is specified) | - | Each item follows the config schema |
request_timeout | Request timeout configuration | number | No | - | - |
custom_host | Route to privately hosted model | string | No | - | Used in combination with provider + api_key |
forward_headers | Forward sensitive headers directly | array of strings | No | - | - |
override_params | Pass model name and other hyper parameters | object | No | ”model”, “temperature”, “frequency_penalty”, “logit_bias”, “logprobs”, “top_logprobs”, “max_tokens”, “n”, “presence_penalty”, “response_format”, “seed”, “stop”, “top_p”, etc. | Pass everything that’s typically part of the payload |
Strategy Object Details
| Key Name | Description | Type | Required | Enum Values | Additional Info |
|---|---|---|---|---|---|
mode | strategy mode for the config | string | Yes | ”single”, “loadbalance”, “fallback”, “conditional” | |
on_status_codes | status codes to apply the strategy. This field is only used when strategy mode is “fallback” | array of numbers | No | Optional |
Cache Object Details
| Key Name | Description | Type | Required | Enum Values | Additional Info |
|---|---|---|---|---|---|
mode | Cache mode | string | Yes | ”simple”, “semantic” | - |
max_age | Maximum age for cache entries | integer | No | - | Optional |
Retry Object Details
| Key Name | Description | Type | Required | Enum Values | Additional Info |
|---|---|---|---|---|---|
attempts | Number of retry attempts | integer | Yes | - | - |
on_status_codes | Status codes to trigger retries | array of strings | No | - | Optional |
use_retry_after_headers | Whether to respect provider’s Retry-After and Retry-After-ms headers | boolean | Default: false |
Circuit Breaker Object Details
| Key Name | Description | Type | Required | Enum Values | Additional Info |
|---|---|---|---|---|---|
failure_threshold | Number of failures after which the circuit opens | number | Yes | - | Minimum value: 1 |
cooldown_interval | Time (in milliseconds) to wait before allowing retries | number | Yes | - | Minimum value: 30000 (30 seconds) |
failure_status_codes | Specific HTTP status codes considered as failures | array of integers | No | - | Optional, defaults to status codes >500 |
Cloud Provider Params (Azure OpenAI, Google Vertex, AWS Bedrock)
Azure OpenAI
| Key Name | Type | Required |
|---|---|---|
azure_resource_name | string | No |
azure_deployment_id | string | No |
azure_api_version | string | No |
azure_model_name | string | No |
Authorization | string (“Bearer $API_KEY”) | No |
Google Vertex AI
| Key Name | Type | Required |
|---|---|---|
vertex_project_id | string | No |
vertex_region | string | No |
AWS Bedrock
| Key Name | Type | Required |
|---|---|---|
aws_access_key_id | string | No |
aws_secret_access_key | string | No |
aws_region | string | No |
aws_session_token | string | No |
Notes
- The strategy
modekey determines the operational mode of the config. If strategymodeis not specified, a single provider mode is assumed, requiring eitherproviderandapi_keyorvirtual_key. - In
loadbalanceandfallbackmodes, thetargetsarray specifies the configurations for each target. - The
cacheandretryobjects provide additional configurations for caching and retry policies, respectively.
Examples
| Example | Description |
|---|---|
| Single Provider | Basic setup with provider slug (recommended) |
| With Hyperparameters | Override model and parameters |
| Cache + Retry | Enable caching and retry logic |
| Load Balancing | Distribute traffic across providers |
| Fallback | Automatic failover between providers |
| Combined Strategies | Nested load balancing with fallback |
| Legacy: API Key | Direct API key usage |
| Legacy: Virtual Key | Virtual key usage |
Single Provider with Provider Slug
The simplest and recommended way to configure a provider.{
"provider": "@openai-prod"
}
Provider Slug with Model & Hyperparameters
Override the model and set custom parameters.{
"provider": "@anthropic-prod",
"override_params": {
"model": "claude-sonnet-4-20250514",
"max_tokens": 512,
"temperature": 0
}
}
Provider Slug with Cache and Retry
Enable semantic caching and retry on specific status codes.{
"provider": "@openai-prod",
"cache": {
"mode": "semantic",
"max_age": 10000
},
"retry": {
"attempts": 5,
"on_status_codes": [429]
}
}
Load Balancing with Provider Slugs
Distribute traffic across multiple providers.{
"strategy": {
"mode": "loadbalance"
},
"targets": [
{ "provider": "@openai-prod" },
{ "provider": "@openai-backup" }
]
}
Fallback Across Providers
Automatically failover to backup providers on errors.{
"strategy": {
"mode": "fallback"
},
"targets": [
{ "provider": "@openai-prod", "override_params": { "model": "gpt-4o" } },
{ "provider": "@anthropic-prod", "override_params": { "model": "claude-sonnet-4-20250514" } }
]
}
Load Balancing and Fallback Combination
Nest strategies for complex routing logic.{
"strategy": {
"mode": "loadbalance"
},
"targets": [
{ "provider": "@openai-prod" },
{
"strategy": {
"mode": "fallback",
"on_status_codes": [429, 500]
},
"targets": [
{ "provider": "@openai-backup" },
{ "provider": "@anthropic-prod" }
]
}
]
}
Single Provider with Provider API Key
OpenAI:{
"provider": "openai",
"api_key": "OPENAI_API_KEY"
}
{
"provider": "anthropic",
"api_key": "ANTHROPIC_API_KEY"
}
Legacy: Single Provider with Virtual Key
Still Supported but we recommend using the provider slug instead
{
"virtual_key": "***"
}