Create automation rule evaluator

POST/v1/private/automations/evaluatorsCreate automation rule evaluator

Request body

application/json

objectAutomationRuleEvaluator_Write

actionstringrequired

one of "evaluator", "annotation_queue_router"

enabledboolean

namestringrequired

maxLength 150 · minLength 0

project_idstring · uuid

Primary project ID (legacy field for backwards compatibility)

project_idsarray of string · uuid

Project IDs for write operations (used when creating/updating rules)

uniqueItems true

Show child attributes

uniqueItems true

sampling_ratenumber · float

Fraction of production (SDK-logged) items this rule scores, from 0 to 1. Trace rules ignore this value for experiment, playground and optimization traces and score them in full; span and thread rules only ever evaluate SDK-logged data.

maximum 1 · minimum 0

trigger_scopestring

Controls whether the rule fires on production traces, experiment traces, or both. Defaults to 'production' if omitted.

one of "production", "experiment", "both"

typestringrequired

one of "llm_as_judge", "user_defined_metric_python", "trace_thread_llm_as_judge", "trace_thread_user_defined_metric_python", "span_llm_as_judge", "span_user_defined_metric_python"

Example request

{
  "action": "annotation_queue_router",
  "enabled": true,
  "name": "string",
  "project_id": "00000000-0000-0000-0000-000000000000",
  "project_ids": [
    "00000000-0000-0000-0000-000000000000"
  ],
  "sampling_rate": 0,
  "trigger_scope": "both",
  "type": "llm_as_judge"
}

Responses

Headers

Location

Examples

curl -X POST 'http://localhost:5173/api/v1/private/automations/evaluators' \
  -H 'Content-Type: application/json' \
  -d '{"action":"annotation_queue_router","enabled":true,"name":"string","project_id":"00000000-0000-0000-0000-000000000000","project_ids":["00000000-0000-0000-0000-000000000000"],"sampling_rate":0,"trigger_scope":"both","type":"llm_as_judge"}'

const response = await fetch('http://localhost:5173/api/v1/private/automations/evaluators', {
  method: 'POST',
  headers: { 'Content-Type': 'application/json' },
  body: JSON.stringify({"action":"annotation_queue_router","enabled":true,"name":"string","project_id":"00000000-0000-0000-0000-000000000000","project_ids":["00000000-0000-0000-0000-000000000000"],"sampling_rate":0,"trigger_scope":"both","type":"llm_as_judge"}),
});
const data = await response.json();

const response = await fetch('http://localhost:5173/api/v1/private/automations/evaluators', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: "{\"action\":\"annotation_queue_router\",\"enabled\":true,\"name\":\"string\",\"project_id\":\"00000000-0000-0000-0000-000000000000\",\"project_ids\":[\"00000000-0000-0000-0000-000000000000\"],\"sampling_rate\":0,\"trigger_scope\":\"both\",\"type\":\"llm_as_judge\"}" });
console.log(await response.json());

import requests

response = requests.request(
    "POST",
    "http://localhost:5173/api/v1/private/automations/evaluators",
    headers={"Content-Type": "application/json"},
    data='{"action":"annotation_queue_router","enabled":true,"name":"string","project_id":"00000000-0000-0000-0000-000000000000","project_ids":["00000000-0000-0000-0000-000000000000"],"sampling_rate":0,"trigger_scope":"both","type":"llm_as_judge"}',
)
print(response.json())

package main

import (
	"context"
	"net/http"
	"strings"
)

func main() {
	ctx := context.Background()
	body := strings.NewReader(`{"action":"annotation_queue_router","enabled":true,"name":"string","project_id":"00000000-0000-0000-0000-000000000000","project_ids":["00000000-0000-0000-0000-000000000000"],"sampling_rate":0,"trigger_scope":"both","type":"llm_as_judge"}`)
	req, err := http.NewRequestWithContext(ctx, "POST", "http://localhost:5173/api/v1/private/automations/evaluators", body)
	if err != nil {
		panic(err)
	}
	req.Header.Set("Content-Type", "application/json")
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		panic(err)
	}
	defer resp.Body.Close()
}

<?php
$request = curl_init("http://localhost:5173/api/v1/private/automations/evaluators");
curl_setopt($request, CURLOPT_CUSTOMREQUEST, "POST");
curl_setopt($request, CURLOPT_POSTFIELDS, "{\"action\":\"annotation_queue_router\",\"enabled\":true,\"name\":\"string\",\"project_id\":\"00000000-0000-0000-0000-000000000000\",\"project_ids\":[\"00000000-0000-0000-0000-000000000000\"],\"sampling_rate\":0,\"trigger_scope\":\"both\",\"type\":\"llm_as_judge\"}");
curl_setopt($request, CURLOPT_HTTPHEADER, ['Content-Type: application/json']);
$response = curl_exec($request);
curl_close($request);

var request = HttpRequest.newBuilder(URI.create("http://localhost:5173/api/v1/private/automations/evaluators"))
    .method("POST", HttpRequest.BodyPublishers.noBody())
    .header("Content-Type", "application/json")
    .method("POST", HttpRequest.BodyPublishers.ofString("{\"action\":\"annotation_queue_router\",\"enabled\":true,\"name\":\"string\",\"project_id\":\"00000000-0000-0000-0000-000000000000\",\"project_ids\":[\"00000000-0000-0000-0000-000000000000\"],\"sampling_rate\":0,\"trigger_scope\":\"both\",\"type\":\"llm_as_judge\"}"))
    .build();
var response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString());

require 'net/http'

uri = URI("http://localhost:5173/api/v1/private/automations/evaluators")
request = Net::HTTPGenericRequest.new("POST", true, true, uri)
request['Content-Type'] = 'application/json'
request.body = "{\"action\":\"annotation_queue_router\",\"enabled\":true,\"name\":\"string\",\"project_id\":\"00000000-0000-0000-0000-000000000000\",\"project_ids\":[\"00000000-0000-0000-0000-000000000000\"],\"sampling_rate\":0,\"trigger_scope\":\"both\",\"type\":\"llm_as_judge\"}"
response = Net::HTTP.start(uri.hostname, uri.port, use_ssl: true) { |http| http.request(request) }

using var client = new HttpClient();
using var request = new HttpRequestMessage(HttpMethod.Post, "http://localhost:5173/api/v1/private/automations/evaluators");
request.Content = new StringContent("{\"action\":\"annotation_queue_router\",\"enabled\":true,\"name\":\"string\",\"project_id\":\"00000000-0000-0000-0000-000000000000\",\"project_ids\":[\"00000000-0000-0000-0000-000000000000\"],\"sampling_rate\":0,\"trigger_scope\":\"both\",\"type\":\"llm_as_judge\"}", Encoding.UTF8, "application/json");
using var response = await client.SendAsync(request);

(no response body — HTTP 201)