Evaluating a workflow

curl -X POST "https://api.anyformat.ai/v3/workflows/$WORKFLOW_ID/dataset/upload/" \
  -H "Authorization: Bearer $ANYFORMAT_API_KEY" \
  -H 'Idempotency-Key: 5f2b6c3e-8a1d-4f7e-9c0b-1a2b3c4d5e6f' \
  -F 'files=@/path/to/invoice_001.pdf' \
  -F 'ground_truth={"invoice_no": "INV-1", "total": "90.00"}'

from anyformat.sdk import Client, DatasetDocument

client = Client(api_key="YOUR_API_KEY")
workflow_id = "0686bb97-8c30-70f0-8000-97669e000eb8"

# One atomic call per document. There is no batch endpoint.
uploads = client.upload_documents_to_dataset(
    workflow_id,
    [
        DatasetDocument(file="invoice_001.pdf", ground_truth={"invoice_no": "INV-1", "total": "90.00"}),
        DatasetDocument(file="invoice_002.pdf", ground_truth={"invoice_no": "INV-2", "total": "120.00"}),
    ],
)
for u in uploads:
    print(u.document_packet_id, u.ground_truth_saved)

interface DatasetUpload {
  document_packet_id: string;
  files: { id: string; name: string; original_name: string | null }[];
  ground_truth_saved: boolean;
}

const workflowId = '0686bb97-8c30-70f0-8000-97669e000eb8';
const formData = new FormData();
formData.append('files', fileInput.files[0]);
formData.append('ground_truth', JSON.stringify({ invoice_no: 'INV-1', total: '90.00' }));

const response = await fetch(
  `https://api.anyformat.ai/v3/workflows/${workflowId}/dataset/upload/`,
  {
    method: 'POST',
    headers: {
      'Authorization': 'Bearer YOUR_API_KEY',
      'Idempotency-Key': crypto.randomUUID(),
    },
    body: formData,
  }
);
if (!response.ok) throw new Error(`API error: ${response.status}`);

const upload: DatasetUpload = await response.json();
console.log(upload.document_packet_id, upload.ground_truth_saved);

curl -X POST "https://api.anyformat.ai/v3/workflows/$WORKFLOW_ID/evals/" \
  -H "Authorization: Bearer $ANYFORMAT_API_KEY" \
  -H 'Idempotency-Key: 9e8d7c6b-5a4f-3e2d-1c0b-a9b8c7d6e5f4'

launched = client.launch_eval(
    workflow_id,
    idempotency_key="9e8d7c6b-5a4f-3e2d-1c0b-a9b8c7d6e5f4",
)
print(launched.eval_id, launched.run_number, launched.enqueued_count)

interface EvalLaunched {
  eval_id: string;
  run_number: number;
  enqueued_count: number;
  failed_count: number;
}

const response = await fetch(
  `https://api.anyformat.ai/v3/workflows/${workflowId}/evals/`,
  {
    method: 'POST',
    headers: {
      'Authorization': 'Bearer YOUR_API_KEY',
      'Content-Type': 'application/json',
      'Idempotency-Key': crypto.randomUUID(),
    },
    // Omit the body to evaluate the current version.
    body: JSON.stringify({}),
  }
);
if (!response.ok) throw new Error(`API error: ${response.status}`);

const launched: EvalLaunched = await response.json();
console.log(launched.eval_id, launched.run_number);

# Repeat until "status" is "processed" or "error".
curl -X GET "https://api.anyformat.ai/v3/workflows/$WORKFLOW_ID/evals/$EVAL_ID/" \
  -H "Authorization: Bearer $ANYFORMAT_API_KEY"

import time

for _ in range(100):
    an_eval = client.get_eval(workflow_id, launched.eval_id)
    if an_eval.status == "processed":
        print(f"accuracy={an_eval.accuracy} "
              f"({an_eval.matched}/{an_eval.mismatched}/{an_eval.ungraded})")
        break
    if an_eval.status == "error":
        raise RuntimeError("Eval failed")
    time.sleep(5)

interface EvalDetail {
  id: string;
  run_number: number;
  version_id: string;
  status: 'in_progress' | 'processed' | 'error';
  accuracy: number | null;
  matched: number | null;
  mismatched: number | null;
  ungraded: number | null;
  file_count: number;
  failed_count: number;
  created_at: string | null;
}

const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));

async function pollEval(evalId: string): Promise<EvalDetail> {
  for (let i = 0; i < 100; i++) {
    const res = await fetch(
      `https://api.anyformat.ai/v3/workflows/${workflowId}/evals/${evalId}/`,
      { headers: { 'Authorization': 'Bearer YOUR_API_KEY' } }
    );
    if (!res.ok) throw new Error(`API error: ${res.status}`);
    const detail: EvalDetail = await res.json();
    if (detail.status !== 'in_progress') return detail;
    await sleep(5000);
  }
  throw new Error('Eval still in progress after timeout');
}

curl -X GET "https://api.anyformat.ai/v3/workflows/$WORKFLOW_ID/evals/?limit=20" \
  -H "Authorization: Bearer $ANYFORMAT_API_KEY"

# iter_evals follows next_cursor across pages for you.
for an_eval in client.iter_evals(workflow_id):
    print(an_eval.run_number, an_eval.status, an_eval.accuracy)

interface EvalPage {
  items: EvalDetail[];
  next_cursor: string | null;
}

let cursor: string | null = null;
do {
  const base = `https://api.anyformat.ai/v3/workflows/${workflowId}/evals/`;
  const url = cursor ? `${base}?cursor=${cursor}` : base;
  const res = await fetch(url, { headers: { 'Authorization': 'Bearer YOUR_API_KEY' } });
  if (!res.ok) throw new Error(`API error: ${res.status}`);
  const page: EvalPage = await res.json();
  for (const e of page.items) console.log(e.run_number, e.status, e.accuracy);
  cursor = page.next_cursor;
} while (cursor);