Complete LLM & AI Model Guide

// lib/ai/ab-testing.ts
import { generateText } from 'ai'

interface ABTestConfig {
  models: {
    variant: 'A' | 'B' | 'C'
    modelId: string
    trafficPercentage: number
  }[]
  metrics: {
    trackQuality: boolean
    trackLatency: boolean
    trackCost: boolean
  }
}

const abConfig: ABTestConfig = {
  models: [
    { variant: 'A', modelId: 'gpt-4o-mini', trafficPercentage: 50 },       // Control
    { variant: 'B', modelId: 'claude-3-haiku', trafficPercentage: 25 },    // Test 1
    { variant: 'C', modelId: 'together/llama-3.3-70b', trafficPercentage: 25 } // Test 2
  ],
  metrics: {
    trackQuality: true,
    trackLatency: true,
    trackCost: true
  }
}

export async function generateWithABTest(
  prompt: string,
  userId: string,
  taskType: string
) {
  // 1. Select model variant based on traffic split
  const variant = selectVariant(abConfig.models, userId)
  const startTime = Date.now()
  
  // 2. Generate with selected model
  const { text } = await generateText({
    model: variant.modelId,
    prompt: prompt,
  })
  
  const latency = Date.now() - startTime
  
  // 3. Track metrics
  await trackABTestMetrics({
    userId,
    taskType,
    variant: variant.variant,
    modelId: variant.modelId,
    latency,
    cost: calculateCost(variant.modelId, prompt.length, text.length),
    timestamp: new Date()
  })
  
  return { text, variant: variant.variant }
}

function selectVariant(models: any[], userId: string) {
  // Consistent hashing: same user always gets same variant
  const hash = hashString(userId)
  const percentage = hash % 100
  
  let cumulative = 0
  for (const model of models) {
    cumulative += model.trafficPercentage
    if (percentage < cumulative) {
      return model
    }
  }
  
  return models[0] // Fallback
}

// Quality evaluation
export async function evaluateResponse(
  response: string,
  expectedCriteria: string[]
) {
  // Use a judge model (e.g., GPT-4o) to evaluate quality
  const { text } = await generateText({
    model: 'gpt-4o',
    prompt: `Evaluate this AI response on these criteria: ${expectedCriteria.join(', ')}.
    
Response: ${response}

Rate each criterion 1-10 and provide overall score.`
  })
  
  return parseEvaluationScore(text)
}

// Example: Comparing models for customer support
async function testCustomerSupportModels() {
  const testCases = [
    "How do I reset my password?",
    "What's your refund policy?",
    "I'm having trouble with checkout"
  ]
  
  const models = ['gpt-4o-mini', 'claude-3-haiku', 'together/llama-3.3-70b']
  
  for (const testCase of testCases) {
    console.log(`Testing: ${testCase}`)
    
    for (const model of models) {
      const start = Date.now()
      const { text } = await generateText({ model, prompt: testCase })
      const latency = Date.now() - start
      
      const quality = await evaluateResponse(text, [
        'Accuracy',
        'Helpfulness',
        'Tone',
        'Conciseness'
      ])
      
      console.log(`  ${model}: ${quality}/10 quality, ${latency}ms, $${calculateCost(model, testCase, text)}`)
    }
  }
}