Self-Improving AI Agents: Building Systems That Learn From Their Mistakes

import Anthropic from "@anthropic-ai/sdk";

const client = new Anthropic();

interface Reflection {
  task: string;
  outcome: "success" | "failure" | "partial";
  lessons: string[];
  confidence: number;
  timestamp: string;
}

async function executeWithReflection(
  task: string,
  context: string
): Promise<{ result: string; reflection: Reflection }> {
  // Step 1: Execute the task
  const executionResult = await client.messages.create({
    model: "claude-sonnet-4-6-20260409",
    max_tokens: 4096,
    system: context,
    messages: [{ role: "user", content: task }],
  });

  const result =
    executionResult.content[0].type === "text"
      ? executionResult.content[0].text
      : "";

  // Step 2: Evaluate the outcome
  const evaluationResult = await client.messages.create({
    model: "claude-sonnet-4-6-20260409",
    max_tokens: 1024,
    messages: [
      {
        role: "user",
        content: `Evaluate this task execution:

Task: ${task}
Result: ${result}

Rate the outcome as "success", "failure", or "partial".
Extract 1-3 specific, actionable lessons learned.
Rate your confidence in each lesson from 0.0 to 1.0.

Respond as JSON:
{
  "outcome": "success|failure|partial",
  "lessons": ["lesson 1", "lesson 2"],
  "confidence": 0.85
}`,
      },
    ],
  });

  const evaluation = JSON.parse(
    evaluationResult.content[0].type === "text"
      ? evaluationResult.content[0].text
      : "{}"
  );

  const reflection: Reflection = {
    task,
    outcome: evaluation.outcome,
    lessons: evaluation.lessons,
    confidence: evaluation.confidence,
    timestamp: new Date().toISOString(),
  };

  return { result, reflection };
}

import { readFile, writeFile } from "fs/promises";

const MEMORY_PATH = ".agent/memory.json";

interface MemoryStore {
  reflections: Reflection[];
  skills: Skill[];
  corrections: Correction[];
}

async function loadMemory(): Promise<MemoryStore> {
  try {
    const data = await readFile(MEMORY_PATH, "utf-8");
    return JSON.parse(data);
  } catch {
    return { reflections: [], skills: [], corrections: [] };
  }
}

async function saveReflection(reflection: Reflection): Promise<void> {
  const memory = await loadMemory();
  memory.reflections.push(reflection);

  // Prune low-confidence reflections when memory gets large
  if (memory.reflections.length > 200) {
    memory.reflections = memory.reflections
      .sort((a, b) => b.confidence - a.confidence)
      .slice(0, 150);
  }

  await writeFile(MEMORY_PATH, JSON.stringify(memory, null, 2));
}

async function getRelevantMemories(task: string): Promise<string> {
  const memory = await loadMemory();

  // Simple keyword matching for retrieval
  const words = task.toLowerCase().split(/\s+/);
  const relevant = memory.reflections.filter((r) =>
    words.some(
      (w) =>
        r.task.toLowerCase().includes(w) ||
        r.lessons.some((l) => l.toLowerCase().includes(w))
    )
  );

  return relevant
    .slice(0, 10)
    .map(
      (r) =>
        `[${r.outcome}] ${r.task}: ${r.lessons.join("; ")} (confidence: ${r.confidence})`
    )
    .join("\n");
}

interface StructuredMemory {
  technical: {
    bugs: Reflection[];
    patterns: Reflection[];
    performance: Reflection[];
  };
  process: {
    planning: Reflection[];
    testing: Reflection[];
    deployment: Reflection[];
  };
  domain: {
    [key: string]: Reflection[];
  };
}

async function categorizeAndStore(reflection: Reflection): Promise<void> {
  const memory = await loadStructuredMemory();

  // Use the model to categorize the reflection
  const categoryResult = await client.messages.create({
    model: "claude-sonnet-4-6-20260409",
    max_tokens: 256,
    messages: [
      {
        role: "user",
        content: `Categorize this lesson into one category.

Lesson: ${reflection.lessons.join("; ")}
Task: ${reflection.task}

Categories:
- technical/bugs: Bug fixes and error handling
- technical/patterns: Code patterns and architecture
- technical/performance: Performance optimization
- process/planning: Task planning and decomposition
- process/testing: Testing strategies
- process/deployment: Deployment and operations
- domain/{topic}: Domain-specific knowledge

Respond with just the category path, e.g., "technical/bugs"`,
      },
    ],
  });

  const category =
    categoryResult.content[0].type === "text"
      ? categoryResult.content[0].text.trim()
      : "domain/general";

  // Store in the appropriate category
  const [top, sub] = category.split("/");
  if (!memory[top]) memory[top] = {};
  if (!memory[top][sub]) memory[top][sub] = [];
  memory[top][sub].push(reflection);

  await saveStructuredMemory(memory);
}

async function retrieveByCategory(
  categories: string[]
): Promise<Reflection[]> {
  const memory = await loadStructuredMemory();
  const results: Reflection[] = [];

  for (const category of categories) {
    const [top, sub] = category.split("/");
    if (memory[top]?.[sub]) {
      results.push(...memory[top][sub]);
    }
  }

  return results.sort((a, b) => b.confidence - a.confidence).slice(0, 20);
}

interface EmbeddedReflection extends Reflection {
  embedding: number[];
}

async function embedReflection(
  reflection: Reflection
): Promise<EmbeddedReflection> {
  const text = `${reflection.task} ${reflection.lessons.join(" ")}`;

  // Using a local embedding model or API
  const embedding = await getEmbedding(text);

  return { ...reflection, embedding };
}

async function findSimilar(
  query: string,
  topK: number = 5
): Promise<Reflection[]> {
  const queryEmbedding = await getEmbedding(query);
  const allMemories = await loadEmbeddedMemories();

  // Cosine similarity search
  const scored = allMemories.map((m) => ({
    reflection: m,
    score: cosineSimilarity(queryEmbedding, m.embedding),
  }));

  return scored
    .sort((a, b) => b.score - a.score)
    .slice(0, topK)
    .map((s) => s.reflection);
}

function cosineSimilarity(a: number[], b: number[]): number {
  let dotProduct = 0;
  let normA = 0;
  let normB = 0;
  for (let i = 0; i < a.length; i++) {
    dotProduct += a[i] * b[i];
    normA += a[i] * a[i];
    normB += b[i] * b[i];
  }
  return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
}

interface Skill {
  name: string;
  description: string;
  steps: string[];
  constraints: string[];
  confidence: number;
  sourceReflections: number;
  lastUpdated: string;
}

async function extractSkill(
  reflections: Reflection[],
  domain: string
): Promise<Skill> {
  const reflectionText = reflections
    .map(
      (r) =>
        `Task: ${r.task}\nOutcome: ${r.outcome}\nLessons: ${r.lessons.join("; ")}`
    )
    .join("\n\n");

  const result = await client.messages.create({
    model: "claude-sonnet-4-6-20260409",
    max_tokens: 2048,
    messages: [
      {
        role: "user",
        content: `Analyze these ${reflections.length} related experiences in the "${domain}" domain and extract a reusable skill.

${reflectionText}

Create a skill with:
1. A clear name (verb + noun, e.g., "Debug Database Connections")
2. A one-sentence description
3. Ordered steps that work reliably
4. Constraints (things to avoid, based on failures)
5. Confidence level (0.0-1.0) based on how many successes vs failures

Respond as JSON:
{
  "name": "...",
  "description": "...",
  "steps": ["step 1", "step 2"],
  "constraints": ["never do X", "always check Y"],
  "confidence": 0.85
}`,
      },
    ],
  });

  const skillData = JSON.parse(
    result.content[0].type === "text" ? result.content[0].text : "{}"
  );

  return {
    ...skillData,
    sourceReflections: reflections.length,
    lastUpdated: new Date().toISOString(),
  };
}

async function evolveSkill(
  existing: Skill,
  newReflections: Reflection[]
): Promise<Skill> {
  const reflectionText = newReflections
    .map(
      (r) =>
        `Task: ${r.task}\nOutcome: ${r.outcome}\nLessons: ${r.lessons.join("; ")}`
    )
    .join("\n\n");

  const result = await client.messages.create({
    model: "claude-sonnet-4-6-20260409",
    max_tokens: 2048,
    messages: [
      {
        role: "user",
        content: `Update this existing skill based on new experiences.

Current skill:
Name: ${existing.name}
Steps: ${existing.steps.join("\n")}
Constraints: ${existing.constraints.join("\n")}
Confidence: ${existing.confidence}
Based on: ${existing.sourceReflections} experiences

New experiences:
${reflectionText}

Update the skill:
- Add new steps if the new experiences reveal missing procedures
- Add new constraints if failures reveal new pitfalls
- Remove steps that new experience shows are unnecessary
- Adjust confidence based on success/failure ratio
- Keep what works, fix what doesn't

Respond as the updated skill JSON.`,
      },
    ],
  });

  const updated = JSON.parse(
    result.content[0].type === "text" ? result.content[0].text : "{}"
  );

  return {
    ...updated,
    sourceReflections: existing.sourceReflections + newReflections.length,
    lastUpdated: new Date().toISOString(),
  };
}

interface SkillLibrary {
  skills: Map<string, Skill>;
  index: Map<string, string[]>; // keyword -> skill names
}

async function findApplicableSkills(
  task: string,
  library: SkillLibrary
): Promise<Skill[]> {
  const words = task.toLowerCase().split(/\s+/);
  const candidateNames = new Set<string>();

  for (const word of words) {
    const matches = library.index.get(word) || [];
    matches.forEach((name) => candidateNames.add(name));
  }

  const candidates = Array.from(candidateNames)
    .map((name) => library.skills.get(name))
    .filter((s): s is Skill => s !== undefined);

  // Sort by confidence, then by recency
  return candidates
    .sort(
      (a, b) =>
        b.confidence - a.confidence ||
        new Date(b.lastUpdated).getTime() - new Date(a.lastUpdated).getTime()
    )
    .slice(0, 3);
}

interface Correction {
  context: string;
  agentBehavior: string;
  humanCorrection: string;
  category: string;
  timestamp: string;
  applied: boolean;
}

async function processCorrection(
  context: string,
  agentBehavior: string,
  humanCorrection: string
): Promise<Correction> {
  // Categorize the correction
  const categoryResult = await client.messages.create({
    model: "claude-sonnet-4-6-20260409",
    max_tokens: 256,
    messages: [
      {
        role: "user",
        content: `Categorize this correction:
Agent did: ${agentBehavior}
Human corrected to: ${humanCorrection}

Categories: style, logic, architecture, security, performance, convention, other
Respond with just the category.`,
      },
    ],
  });

  const category =
    categoryResult.content[0].type === "text"
      ? categoryResult.content[0].text.trim()
      : "other";

  const correction: Correction = {
    context,
    agentBehavior,
    humanCorrection,
    category,
    timestamp: new Date().toISOString(),
    applied: false,
  };

  await storeCorrection(correction);

  // Check if we have enough corrections in this category to update a skill
  const categoryCorrections = await getCorrectionsByCategory(category);
  if (categoryCorrections.length >= 3) {
    await updateSkillFromCorrections(category, categoryCorrections);
  }

  return correction;
}

function correctionWeight(correction: Correction): number {
  const ageInDays =
    (Date.now() - new Date(correction.timestamp).getTime()) /
    (1000 * 60 * 60 * 24);

  // Half-life of 30 days
  return Math.exp(-0.693 * (ageInDays / 30));
}

async function getWeightedCorrections(
  category: string
): Promise<Correction[]> {
  const corrections = await getCorrectionsByCategory(category);
  return corrections
    .map((c) => ({ ...c, weight: correctionWeight(c) }))
    .filter((c) => c.weight > 0.1) // Discard corrections older than ~100 days
    .sort((a, b) => b.weight - a.weight);
}

async function selfImprovingAgent(task: string): Promise<string> {
  // 1. Load relevant context
  const memories = await getRelevantMemories(task);
  const skills = await findApplicableSkills(task, await loadSkillLibrary());
  const corrections = await getRecentCorrections(task);

  // 2. Build enhanced context
  const context = buildContext(memories, skills, corrections);

  // 3. Execute with enhanced context
  const { result, reflection } = await executeWithReflection(task, context);

  // 4. Store the reflection
  await saveReflection(reflection);

  // 5. Check if any skills need updating
  if (reflection.lessons.length > 0) {
    const relatedReflections = await findRelatedReflections(reflection);
    if (relatedReflections.length >= 5) {
      const existingSkill = await findMatchingSkill(task);
      if (existingSkill) {
        await evolveSkill(existingSkill, [reflection]);
      } else {
        const newSkill = await extractSkill(relatedReflections, task);
        await addToSkillLibrary(newSkill);
      }
    }
  }

  return result;
}

function buildContext(
  memories: string,
  skills: Skill[],
  corrections: Correction[]
): string {
  let context = "You are an AI agent that learns from experience.\n\n";

  if (skills.length > 0) {
    context += "## Relevant Skills\n\n";
    for (const skill of skills) {
      context += `### ${skill.name}\n`;
      context += `${skill.description}\n`;
      context += `Steps:\n${skill.steps.map((s, i) => `${i + 1}. ${s}`).join("\n")}\n`;
      context += `Constraints:\n${skill.constraints.map((c) => `- ${c}`).join("\n")}\n\n`;
    }
  }

  if (memories) {
    context += "## Relevant Past Experiences\n\n";
    context += memories + "\n\n";
  }

  if (corrections.length > 0) {
    context += "## Recent Corrections\n\n";
    for (const c of corrections.slice(0, 5)) {
      context += `- Instead of: ${c.agentBehavior}\n  Do: ${c.humanCorrection}\n`;
    }
    context += "\n";
  }

  return context;
}

interface PerformanceMetrics {
  totalTasks: number;
  firstAttemptSuccesses: number;
  correctionsReceived: number;
  averageConfidence: number;
  skillCount: number;
}

function successRate(metrics: PerformanceMetrics): number {
  return metrics.firstAttemptSuccesses / metrics.totalTasks;
}

interface Override {
  skillName: string;
  originalBehavior: string;
  overrideBehavior: string;
  reason: string;
  permanent: boolean;
}