Evals

import { Agentbase } from '@agentbase/sdk';

const agentbase = new Agentbase({
  apiKey: process.env.AGENTBASE_API_KEY
});

// Define an eval case
interface EvalCase {
  name: string;
  input: string;
  expectedBehavior: string;
  validate: (output: string) => boolean;
}

// Create eval case
const evalCase: EvalCase = {
  name: "Basic Math",
  input: "What is 15 multiplied by 23?",
  expectedBehavior: "Should calculate and return 345",
  validate: (output: string) => {
    return output.includes('345');
  }
};

// Run eval
async function runEval(evalCase: EvalCase): Promise<boolean> {
  const result = await agentbase.runAgent({
    message: evalCase.input,
    mode: "base"
  });

  const passed = evalCase.validate(result.message);

  console.log(`Eval: ${evalCase.name}`);
  console.log(`Input: ${evalCase.input}`);
  console.log(`Output: ${result.message}`);
  console.log(`Result: ${passed ? '✅ PASS' : '❌ FAIL'}`);

  return passed;
}

// Execute
const passed = await runEval(evalCase);

// Comprehensive eval suite
class EvalSuite {
  private results: Map<string, boolean> = new Map();

  async run(evalCases: EvalCase[]): Promise<void> {
    console.log(`\n🧪 Running ${evalCases.length} evals...\n`);

    for (const evalCase of evalCases) {
      try {
        const passed = await this.runSingleEval(evalCase);
        this.results.set(evalCase.name, passed);
      } catch (error) {
        console.error(`Error in eval ${evalCase.name}:`, error);
        this.results.set(evalCase.name, false);
      }
    }

    this.printSummary();
  }

  async runSingleEval(evalCase: EvalCase): Promise<boolean> {
    const result = await agentbase.runAgent({
      message: evalCase.input,
      mode: "base"
    });

    const passed = evalCase.validate(result.message);

    const status = passed ? '✅ PASS' : '❌ FAIL';
    console.log(`${status} ${evalCase.name}`);

    return passed;
  }

  printSummary(): void {
    const total = this.results.size;
    const passed = Array.from(this.results.values()).filter(r => r).length;
    const failed = total - passed;
    const passRate = (passed / total) * 100;

    console.log('\n' + '='.repeat(50));
    console.log('📊 Eval Summary');
    console.log('='.repeat(50));
    console.log(`Total:      ${total}`);
    console.log(`Passed:     ${passed} ✅`);
    console.log(`Failed:     ${failed} ❌`);
    console.log(`Pass Rate:  ${passRate.toFixed(1)}%`);
    console.log('='.repeat(50) + '\n');

    if (passRate < 100) {
      console.log('❌ Some evals failed. Review failures above.\n');
      process.exit(1);
    } else {
      console.log('✅ All evals passed!\n');
    }
  }

  getResults() {
    return this.results;
  }
}

// Define eval cases
const evalCases: EvalCase[] = [
  {
    name: "File Reading",
    input: "Read the file data.csv and show the first row",
    expectedBehavior: "Should read file and display header",
    validate: (output) => output.toLowerCase().includes('header') || output.includes(',')
  },
  {
    name: "Data Calculation",
    input: "Calculate the sum of 123 and 456",
    expectedBehavior: "Should return 579",
    validate: (output) => output.includes('579')
  },
  {
    name: "Error Handling",
    input: "Read nonexistent_file.txt",
    expectedBehavior: "Should handle missing file gracefully",
    validate: (output) => output.toLowerCase().includes('not found') || output.toLowerCase().includes('does not exist')
  }
];

// Run suite
const suite = new EvalSuite();
await suite.run(evalCases);

// Test performance metrics
interface PerformanceEval {
  name: string;
  input: string;
  maxSteps: number;
  maxDuration: number;  // milliseconds
  maxCost: number;      // dollars
}

async function runPerformanceEval(eval_: PerformanceEval): Promise<boolean> {
  const startTime = Date.now();
  let stepCount = 0;
  let totalCost = 0;

  const result = await agentbase.runAgent({
    message: eval_.input,
    mode: "base",
    stream: true
  });

  for await (const event of result) {
    if (event.type === 'agent_step') {
      stepCount++;
    }
    if (event.type === 'agent_cost') {
      totalCost += parseFloat(event.cost);
    }
  }

  const duration = Date.now() - startTime;

  const stepsPass = stepCount <= eval_.maxSteps;
  const durationPass = duration <= eval_.maxDuration;
  const costPass = totalCost <= eval_.maxCost;

  const allPass = stepsPass && durationPass && costPass;

  console.log(`\n${allPass ? '✅' : '❌'} ${eval_.name}`);
  console.log(`  Steps: ${stepCount}/${eval_.maxSteps} ${stepsPass ? '✅' : '❌'}`);
  console.log(`  Duration: ${duration}ms/${eval_.maxDuration}ms ${durationPass ? '✅' : '❌'}`);
  console.log(`  Cost: $${totalCost.toFixed(4)}/$${eval_.maxCost} ${costPass ? '✅' : '❌'}`);

  return allPass;
}

// Performance test cases
const performanceEvals: PerformanceEval[] = [
  {
    name: "Quick Query",
    input: "What is 2+2?",
    maxSteps: 2,
    maxDuration: 3000,
    maxCost: 0.01
  },
  {
    name: "File Processing",
    input: "Read data.csv and count rows",
    maxSteps: 5,
    maxDuration: 10000,
    maxCost: 0.05
  }
];

// Run performance evals
for (const eval_ of performanceEvals) {
  await runPerformanceEval(eval_);
}

// Use LLM to evaluate quality
async function llmAsJudge(
  taskDescription: string,
  agentOutput: string,
  criteria: string[]
): Promise<{ score: number; feedback: string }> {
  const evaluationPrompt = `
Evaluate this agent response on a scale of 1-10 for each criterion.

Task: ${taskDescription}

Agent Response:
${agentOutput}

Evaluation Criteria:
${criteria.map((c, i) => `${i + 1}. ${c}`).join('\n')}

Provide:
1. Score for each criterion (1-10)
2. Overall score (average)
3. Brief feedback explaining the scores

Format as JSON:
{
  "scores": { "criterion1": score, "criterion2": score, ... },
  "overallScore": average,
  "feedback": "explanation"
}
  `;

  const evaluation = await agentbase.runAgent({
    message: evaluationPrompt,
    mode: "base"
  });

  // Parse JSON from response
  const jsonMatch = evaluation.message.match(/\{[\s\S]*\}/);
  if (!jsonMatch) {
    throw new Error('Could not parse evaluation JSON');
  }

  return JSON.parse(jsonMatch[0]);
}

// Quality eval case
async function runQualityEval(
  input: string,
  criteria: string[],
  minScore: number = 7
): Promise<boolean> {
  // Get agent response
  const result = await agentbase.runAgent({
    message: input,
    mode: "base"
  });

  // Evaluate quality
  const evaluation = await llmAsJudge(input, result.message, criteria);

  const passed = evaluation.overallScore >= minScore;

  console.log(`\n${passed ? '✅' : '❌'} Quality Eval`);
  console.log(`  Input: ${input}`);
  console.log(`  Score: ${evaluation.overallScore}/10`);
  console.log(`  Feedback: ${evaluation.feedback}`);

  return passed;
}

// Example usage
await runQualityEval(
  "Explain how photosynthesis works",
  [
    "Accuracy: Is the explanation scientifically correct?",
    "Clarity: Is it easy to understand?",
    "Completeness: Does it cover key concepts?",
    "Examples: Does it include helpful examples?"
  ],
  8.0  // Minimum score
);

// Test that changes don't break existing functionality
class RegressionTester {
  private baseline: Map<string, string> = new Map();

  async createBaseline(testCases: EvalCase[]): Promise<void> {
    console.log('📸 Creating baseline...');

    for (const test of testCases) {
      const result = await agentbase.runAgent({
        message: test.input,
        mode: "base"
      });

      this.baseline.set(test.name, result.message);
    }

    // Save baseline to file
    await this.saveBaseline();
    console.log('✅ Baseline created\n');
  }

  async testRegression(testCases: EvalCase[]): Promise<void> {
    console.log('🔍 Testing for regressions...\n');

    // Load baseline
    await this.loadBaseline();

    let regressions = 0;

    for (const test of testCases) {
      const result = await agentbase.runAgent({
        message: test.input,
        mode: "base"
      });

      const baselineOutput = this.baseline.get(test.name);

      if (!baselineOutput) {
        console.log(`⚠️  ${test.name}: No baseline found`);
        continue;
      }

      // Compare outputs
      const similar = this.compareOutputs(baselineOutput, result.message);

      if (!similar) {
        regressions++;
        console.log(`❌ REGRESSION: ${test.name}`);
        console.log(`   Baseline: ${baselineOutput.substring(0, 100)}...`);
        console.log(`   Current:  ${result.message.substring(0, 100)}...`);
      } else {
        console.log(`✅ ${test.name}`);
      }
    }

    if (regressions > 0) {
      console.log(`\n❌ Found ${regressions} regression(s)`);
      process.exit(1);
    } else {
      console.log('\n✅ No regressions detected');
    }
  }

  compareOutputs(baseline: string, current: string): boolean {
    // Implement comparison logic
    // Could use:
    // - Exact match
    // - Semantic similarity
    // - Key phrase matching
    // - LLM-based comparison

    // Simple example: check if key information is present
    const baselineKeywords = this.extractKeywords(baseline);
    const currentKeywords = this.extractKeywords(current);

    const overlap = baselineKeywords.filter(k => currentKeywords.includes(k));
    const similarity = overlap.length / baselineKeywords.length;

    return similarity > 0.8;  // 80% of keywords should match
  }

  extractKeywords(text: string): string[] {
    // Simple keyword extraction
    return text
      .toLowerCase()
      .split(/\W+/)
      .filter(word => word.length > 4)
      .slice(0, 20);
  }

  async saveBaseline(): Promise<void> {
    const data = Object.fromEntries(this.baseline);
    await fs.writeFile(
      'eval-baseline.json',
      JSON.stringify(data, null, 2)
    );
  }

  async loadBaseline(): Promise<void> {
    try {
      const data = await fs.readFile('eval-baseline.json', 'utf-8');
      const parsed = JSON.parse(data);
      this.baseline = new Map(Object.entries(parsed));
    } catch (error) {
      console.error('Could not load baseline:', error);
    }
  }
}

// Usage
const tester = new RegressionTester();

// Create baseline before making changes
await tester.createBaseline(testCases);

// After making changes, test for regressions
await tester.testRegression(testCases);