Scorecard Format

import { z } from "zod";

const DimensionName = z.enum([
  "api-surface",
  "cli-design",
  "mcp-server",
  "discovery-aeo",
  "authentication",
  "error-handling",
  "tool-design",
  "context-files",
  "testing",
  "retrievability",
]);

const ConfidenceLevel = z.enum(["high", "medium", "low"]);

const EvidenceItem = z.object({
  type: z.enum(["file", "grep", "glob", "command"]),
  description: z.string(),
  location: z.string().describe("File path, grep pattern, glob pattern, or command"),
  result: z.string().describe("Quoted output or finding"),
});

const Finding = z.object({
  id: z.string().describe('Stable ID other Finding arrays reference, e.g. "mcp-not-implemented"'),
  what: z.string().describe("Specific issue with file:line reference"),
  why: z.string().describe("Why this matters for agent consumption"),
  fix: z.string().describe("Concrete steps to resolve"),
  dimension: DimensionName,
  currentScore: z.number().int().min(0).max(3),
  targetScore: z.number().int().min(0).max(3),
  severity: z.enum(["critical", "high", "medium", "low"]),
});

const FindingCluster = z.object({
  name: z.string().describe("Descriptive cluster name"),
  rationale: z.string().describe("Why these findings belong together"),
  findings: z.array(Finding),
  suggestedApproach: z.string().describe("1-2 sentences on how to tackle"),
  dependencies: z.array(z.string()).describe("Other cluster names to complete first"),
});

const Dimension = z.object({
  name: DimensionName,
  score: z.number().int().min(0).max(3).nullable().describe("null if N/A"),
  confidence: ConfidenceLevel.nullable().describe("null if N/A"),
  evidence: z.array(EvidenceItem),
  findings: z.array(z.string()).describe("Finding IDs referencing FindingCluster.findings"),
  notes: z.string().optional(),
});

const OverallRating = z.enum(["human-only", "agent-tolerant", "agent-ready", "agent-first"]);

const Scorecard = z.object({
  version: z.literal("1.0"),
  project: z.object({
    name: z.string(),
    repository: z.string().url().optional(),
    description: z.string().optional(),
  }),
  date: z.string().datetime().describe("ISO 8601, e.g., 2026-04-17T15:30:00Z"),
  auditor: z.string().optional(),
  dimensions: z.array(Dimension).length(10),
  findingClusters: z.array(FindingCluster),
  overall: z.object({
    rawScore: z.number().int().min(0).max(30),
    maxApplicable: z.number().int().min(1).max(30),
    scaledScore: z
      .number()
      .int()
      .min(0)
      .max(30)
      .describe("Raw score scaled to 30 after excluding N/A dimensions"),
    rating: OverallRating,
    percentile: z.number().min(0).max(100).optional(),
  }),
  trend: z
    .object({
      previousDate: z.string().datetime().optional(),
      previousScore: z.number().int().optional(),
      delta: z.number().int().optional(),
      status: z.enum(["improved", "stable", "regressed"]).optional(),
    })
    .optional(),
});

export type Scorecard = z.infer<typeof Scorecard>;

{
  "version": "1.0",
  "project": {
    "name": "acme-widgets",
    "repository": "https://github.com/acme/widgets-api",
    "description": "Customer-facing REST API for SaaS platform"
  },
  "date": "2026-06-10T15:30:00Z",
  "auditor": "surface@1.2.0",
  "dimensions": [
    {
      "name": "api-surface",
      "score": 2,
      "confidence": "high",
      "evidence": [
        {
          "type": "file",
          "description": "OpenAPI spec exists",
          "location": "docs/api/openapi.yaml",
          "result": "40 endpoints, operationId on 40 (100%), descriptions 30-50 words"
        },
        {
          "type": "grep",
          "description": "Agent-oriented descriptions",
          "location": "grep -A 2 'description:' docs/api/openapi.yaml",
          "result": "Sample: 'Search for users by email. Use when you need to find user records.' (agent context present)"
        },
        {
          "type": "grep",
          "description": "Arazzo workflows",
          "location": "grep -r 'Arazzo\\|x-action' docs/",
          "result": "0 matches. No workflow definitions."
        }
      ],
      "findings": [],
      "notes": "OpenAPI is well-documented with agent-oriented descriptions. Missing Arazzo workflows prevent multi-step operation definition."
    },
    {
      "name": "cli-design",
      "score": null,
      "confidence": null,
      "evidence": [],
      "findings": [],
      "notes": "N/A: Project is a web API with no CLI tool"
    },
    {
      "name": "mcp-server",
      "score": 0,
      "confidence": "high",
      "evidence": [
        {
          "type": "glob",
          "description": "MCP server config",
          "location": "glob('**/.mcp.json')",
          "result": "0 files"
        },
        {
          "type": "grep",
          "description": "MCP SDK imports",
          "location": "grep -r '@modelcontextprotocol/sdk' .",
          "result": "0 matches"
        }
      ],
      "findings": ["mcp-not-implemented"],
      "notes": "No MCP server. Project could expose tools to agents via MCP."
    },
    {
      "name": "discovery-aeo",
      "score": 1,
      "confidence": "medium",
      "evidence": [
        {
          "type": "file",
          "description": "AGENTS.md",
          "location": "AGENTS.md (repo root)",
          "result": "45 lines, covers commands and conventions"
        },
        {
          "type": "glob",
          "description": "Discovery files",
          "location": "glob('**/llms.txt')",
          "result": "0 files"
        },
        {
          "type": "grep",
          "description": "JSON-LD markup",
          "location": "grep -r 'application/ld+json' docs/",
          "result": "0 matches"
        }
      ],
      "findings": ["discovery-llms-txt-missing"],
      "notes": "AGENTS.md present but minimal. No llms.txt or structured data for agent discovery."
    },
    {
      "name": "authentication",
      "score": 2,
      "confidence": "high",
      "evidence": [
        {
          "type": "file",
          "description": "OAuth config",
          "location": "src/auth/oauth.ts:10–45",
          "result": "Client credentials grant implemented. Token expires in 1 hour."
        },
        {
          "type": "grep",
          "description": "JWT validation",
          "location": "grep -A 5 'verify' src/auth/jwt.ts",
          "result": "Validates signature, iss, aud, exp claims"
        },
        {
          "type": "grep",
          "description": "Token exchange (RFC 8693)",
          "location": "grep -r 'token-exchange\\|RFC 8693' src/",
          "result": "0 matches"
        }
      ],
      "findings": [],
      "notes": "OAuth 2.1 M2M with scoped tokens. Token exchange (RFC 8693) not applicable to the advertised flows."
    },
    {
      "name": "error-handling",
      "score": 2,
      "confidence": "high",
      "evidence": [
        {
          "type": "file",
          "description": "Error middleware",
          "location": "src/middleware/errors.ts:50–80",
          "result": "Returns RFC 9457 Problem Details: { type, title, status, detail, instance }"
        },
        {
          "type": "grep",
          "description": "is_retriable field",
          "location": "grep -r 'is_retriable\\|isRetriable' src/",
          "result": "is_retriable boolean present on all error types"
        },
        {
          "type": "command",
          "description": "Error response test",
          "location": "curl -s http://localhost:3000/api/users/invalid -H 'Accept: application/json'",
          "result": "{ type: 'https://api.example.com/errors/not-found', title: 'Not Found', status: 404, detail: 'User ID not valid', is_retriable: false }"
        }
      ],
      "findings": ["error-missing-doc-uri"],
      "notes": "RFC 9457 implemented with complete is_retriable coverage and a suggestions array. Missing doc_uri for level 3."
    },
    {
      "name": "tool-design",
      "score": 1,
      "confidence": "medium",
      "evidence": [
        {
          "type": "glob",
          "description": "Tool definitions",
          "location": "glob('**/tools/**')",
          "result": "tools/ directory found with 6 tool files"
        },
        {
          "type": "file",
          "description": "Tool descriptions",
          "location": "tools/search-users.ts:1–20",
          "result": "Description: 'Search for users' (3 words, no agent context)"
        },
        {
          "type": "grep",
          "description": "toModelOutput",
          "location": "grep -r 'toModelOutput' tools/",
          "result": "0 matches"
        }
      ],
      "findings": ["tools-weak-descriptions"],
      "notes": "Basic tool schemas exist but descriptions lack 'use when' context and disambiguation. No token optimization."
    },
    {
      "name": "context-files",
      "score": 2,
      "confidence": "high",
      "evidence": [
        {
          "type": "file",
          "description": "AGENTS.md quality",
          "location": "AGENTS.md",
          "result": "Commands section present with exact invocations. Permission boundaries defined (always/ask-first/never). 45 lines."
        },
        {
          "type": "grep",
          "description": "CLAUDE.md imports AGENTS.md",
          "location": "grep -c '^@AGENTS.md' CLAUDE.md",
          "result": "No CLAUDE.md present - not required for a top score"
        },
        {
          "type": "grep",
          "description": "Cursor rules",
          "location": "glob('**/.cursor/rules/**')",
          "result": "0 files"
        }
      ],
      "findings": [],
      "notes": "Good AGENTS.md. No CLAUDE.md or Cursor rules present - neither is required; add one only if that tool needs something AGENTS.md can't give it."
    },
    {
      "name": "testing",
      "score": 1,
      "confidence": "medium",
      "evidence": [
        {
          "type": "glob",
          "description": "Test files",
          "location": "glob('**/*.test.ts')",
          "result": "8 test files found"
        },
        {
          "type": "grep",
          "description": "MCP server tests",
          "location": "grep -r 'InMemoryTransport' tests/",
          "result": "0 matches (no MCP server to test)"
        },
        {
          "type": "grep",
          "description": "Error recovery tests",
          "location": "grep -r 'isRetriable\\|retry' tests/",
          "result": "2 matches (minimal coverage)"
        },
        {
          "type": "command",
          "description": "Test suite run",
          "location": "npm test -- --coverage",
          "result": "42 tests, 81% coverage. No error recovery or multi-step flow tests."
        }
      ],
      "findings": ["testing-error-recovery-missing"],
      "notes": "Basic tests exist but no agent-specific evaluation. Missing error recovery and multi-step flow testing."
    },
    {
      "name": "retrievability",
      "score": null,
      "confidence": null,
      "evidence": [],
      "findings": [],
      "notes": "N/A: Project does not expose a searchable knowledge or retrieval surface"
    }
  ],
  "findingClusters": [
    {
      "name": "Agent tool discoverability",
      "rationale": "Agents struggle to find and understand tools because tool descriptions are terse, no MCP server exists, and llms.txt is missing.",
      "findings": [
        {
          "id": "tools-weak-descriptions",
          "what": "tools/search-users.ts:1–20: Tool descriptions are terse (3 words) with no 'use when' context",
          "why": "Agents use tool descriptions as prompts. Terse descriptions lead to incorrect tool selection.",
          "fix": "Rewrite descriptions to include: when to use, when NOT to use, required parameters, return type. Target 30-50 words.",
          "dimension": "tool-design",
          "currentScore": 1,
          "targetScore": 2,
          "severity": "high"
        },
        {
          "id": "mcp-not-implemented",
          "what": "mcp-server: 0 files matching **/.mcp.json (no MCP server exists)",
          "why": "Agents consume MCP servers to discover tools. Without MCP, agents must rely on API docs or CLI help.",
          "fix": "Create .mcp.json and MCP server implementation. Expose 6 existing tools via MCP. Use InMemoryTransport for testing.",
          "dimension": "mcp-server",
          "currentScore": 0,
          "targetScore": 2,
          "severity": "critical"
        },
        {
          "id": "discovery-llms-txt-missing",
          "what": "llms.txt missing from web root",
          "why": "Agent web crawlers look for llms.txt to understand project scope and API surface. Without it, discovery is harder.",
          "fix": "Create llms.txt at project root with H2 sections for API, CLI, tools, and links to key docs.",
          "dimension": "discovery-aeo",
          "currentScore": 1,
          "targetScore": 2,
          "severity": "medium"
        }
      ],
      "suggestedApproach": "Start with tool descriptions (1 hour). Add MCP server (4 hours). Add llms.txt (30 min). This cluster improves tool discoverability score from 1→2 across three dimensions.",
      "dependencies": []
    },
    {
      "name": "Error recovery for agents",
      "rationale": "Agents cannot link an error back to documentation, and testing does not cover recovery flows.",
      "findings": [
        {
          "id": "error-missing-doc-uri",
          "what": "src/middleware/errors.ts: Problem Details responses omit doc_uri",
          "why": "Agents use doc_uri to look up the relevant error-reference page without guessing a URL.",
          "fix": "Add a doc_uri field to the Problem Details response, pointing at the matching error-reference page.",
          "dimension": "error-handling",
          "currentScore": 2,
          "targetScore": 3,
          "severity": "medium"
        },
        {
          "id": "testing-error-recovery-missing",
          "what": "tests/: No error recovery tests. grep 'retry' returns 2 matches (minimal).",
          "why": "Without tests, error recovery logic regresses. Agents depend on consistent behavior.",
          "fix": "Add tests for: rate limit → retry, validation error → no retry, timeout → retry. Cover multi-step recovery.",
          "dimension": "testing",
          "currentScore": 1,
          "targetScore": 2,
          "severity": "medium"
        }
      ],
      "suggestedApproach": "Add doc_uri to the error schema (1 hour). Write recovery tests (2 hours). Total: 3 hours.",
      "dependencies": []
    }
  ],
  "overall": {
    "rawScore": 11,
    "maxApplicable": 24,
    "scaledScore": 14,
    "rating": "agent-tolerant",
    "percentile": 35
  },
  "trend": {
    "previousDate": "2026-05-01T10:00:00Z",
    "previousScore": 8,
    "delta": 3,
    "status": "improved"
  }
}

import { Scorecard } from "./scorecard-schema";

async function loadScorecard(filePath: string): Promise<Scorecard> {
  const raw = await Deno.readTextFile(filePath);
  const json = JSON.parse(raw);
  return Scorecard.parse(json); // Zod validation
}

function displaySummary(scorecard: Scorecard): void {
  const { overall, project } = scorecard;
  console.log(`
Project: ${project.name}
Date: ${scorecard.date}
Rating: ${overall.rating} (${overall.scaledScore}/30)

Scores:
${scorecard.dimensions
  .filter((d) => d.score !== null)
  .map((d) => `  ${d.name}: ${d.score}/3 (${d.confidence})`)
  .join("\n")}
  `);
}

function findHighSeverityIssues(scorecard: Scorecard): Finding[] {
  return scorecard.findingClusters
    .flatMap((cluster) => cluster.findings)
    .filter((f) => f.severity === "critical" || f.severity === "high")
    .sort((a, b) => {
      const severity = { critical: 3, high: 2, medium: 1, low: 0 };
      return severity[b.severity] - severity[a.severity];
    });
}

docs/surface/scorecard-[YYYY-MM-DD].json  # Versioned snapshots
docs/surface/scorecard-latest.json        # Current snapshot (symlink or copy)
docs/surface/history.md                   # Human-readable changelog

git log docs/surface/scorecard-*.json