Cut LLM Cost with a Jev-Verified Cascade

import { OpenRouter } from '@openrouter/sdk';

const openrouter = new OpenRouter({ apiKey: process.env.OPENROUTER_API_KEY });

const DRAFT_MODEL = '~openai/gpt-luna-latest';
const ESCALATION_MODEL = '~openai/gpt-astra-latest';

const SYSTEM_PROMPT =
  'You are the support assistant for Northwind. Answer only from the help center excerpts provided. If they do not cover the question, say so and offer to connect the customer with support.';

async function draftAnswer(model: string, question: string, excerpts: string[]): Promise<string> {
  const result = await openrouter.chat.send({
    chatRequest: {
      model,
      messages: [
        { role: 'system', content: SYSTEM_PROMPT },
        {
          role: 'user',
          content: `Help center excerpts:\n${excerpts.map((text, i) => `[${i + 1}] ${text}`).join('\n')}\n\nCustomer question: ${question}`,
        },
      ],
    },
  });
  if (result instanceof ReadableStream) {
    throw new Error('Expected a non-streaming response');
  }
  const content = result.choices[0]?.message.content;
  if (typeof content !== 'string') {
    throw new Error('Expected text content');
  }
  return content;
}

const decisions = new OpenRouter({ apiKey: process.env.OPENROUTER_API_KEY, serverURL: 'https://openrouter.ai' });

type Verdict = {
  choice: 'supported' | 'unsupported' | 'declined';
  confidence: number;
  probabilities: Record<string, number>;
};

function isLabel(choice: string): choice is Verdict['choice'] {
  return choice === 'supported' || choice === 'unsupported' || choice === 'declined';
}

async function verifyAnswer(question: string, excerpts: string[], answer: string): Promise<Verdict> {
  const response = await decisions.alpha.decisions.create({
    decisionsRequest: {
      model: 'typesafe/jev-1.13',
      state: { help_center_excerpts: excerpts, customer_question: question, assistant_answer: answer },
      questions: {
        support: {
          type: 'choice',
          instructions: 'Compare assistant_answer against help_center_excerpts. Which one describes it?',
          criteria: {
            supported: 'The answer addresses customer_question, and every fact, number, and policy in it is stated in the excerpts.',
            unsupported: 'The answer states at least one fact, number, or policy that the excerpts do not contain or that contradicts them, or it answers a different question than customer_question.',
            declined: 'The answer says the excerpts do not cover the question and does not assert facts of its own.',
          },
        },
      },
    },
  });
  const verdict = response.answers.support;
  if (verdict.type !== 'choice' || !isLabel(verdict.choice)) {
    throw new Error('Expected a choice answer with a known label');
  }
  return { choice: verdict.choice, confidence: verdict.confidence ?? 0, probabilities: verdict.probabilities ?? {} };
}

const ACCEPT_CONFIDENCE = 0.8;

type CascadeResult = {
  route: 'send' | 'handoff';
  model: string;
  answer: string;
  verdicts: Verdict[];
};

async function answerWithCascade(question: string, excerpts: string[]): Promise<CascadeResult> {
  const verdicts: Verdict[] = [];
  let answer = '';
  for (const model of [DRAFT_MODEL, ESCALATION_MODEL]) {
    answer = await draftAnswer(model, question, excerpts);
    const verdict = await verifyAnswer(question, excerpts, answer);
    verdicts.push(verdict);
    if (verdict.choice === 'supported' && verdict.confidence >= ACCEPT_CONFIDENCE) {
      return { route: 'send', model, answer, verdicts };
    }
    if (verdict.choice === 'declined') {
      return { route: 'handoff', model, answer, verdicts };
    }
  }
  return { route: 'handoff', model: ESCALATION_MODEL, answer, verdicts };
}

const excerpts = [
  'Plan limits: Free workspaces have up to 5 members and 3 projects. Team workspaces ($8 per member per month, billed annually) have unlimited projects and up to 250 members.',
  'Billing: Team plans are billed annually. Downgrading to Free takes effect at the end of the current billing period. We do not issue prorated refunds for unused time.',
  'Data export: Workspace owners can export all projects as CSV or JSON from Settings > Data. Exports include tasks, comments, and attachment metadata but not attachment files.',
];

const result = await answerWithCascade('How many people can I add on the free plan?', excerpts);

{
  "question": "How many people can I add on the free plan?",
  "route": "send",
  "model": "~openai/gpt-luna-latest",
  "answer": "The Free plan allows up to 5 members per workspace.",
  "verdicts": [
    {
      "choice": "supported",
      "confidence": 1,
      "probabilities": {
        "supported": 1,
        "unsupported": 0,
        "declined": 0
      }
    }
  ]
}

{
  "question": "Do you integrate with Jira?",
  "route": "handoff",
  "model": "~openai/gpt-luna-latest",
  "answer": "The provided help center excerpts don’t mention Jira integrations. I can connect you with support to confirm whether Jira is supported.",
  "verdicts": [
    {
      "choice": "declined",
      "confidence": 0.97,
      "probabilities": {
        "supported": 0,
        "unsupported": 0.02,
        "declined": 0.98
      }
    }
  ]
}

{
  "question": "How many people can I add on the free plan?",
  "route": "send",
  "model": "~openai/gpt-astra-latest",
  "answer": "The Free plan supports up to **5 members** per workspace.",
  "verdicts": [
    {
      "choice": "unsupported",
      "confidence": 1,
      "probabilities": {
        "supported": 0,
        "unsupported": 1,
        "declined": 0
      }
    },
    {
      "choice": "supported",
      "confidence": 1,
      "probabilities": {
        "supported": 1,
        "unsupported": 0,
        "declined": 0
      }
    }
  ]
}