From 8b3d8adde234024831a26af9ea3a3e34d35eaf40 Mon Sep 17 00:00:00 2001 From: Adam Jacob Date: Thu, 19 Mar 2026 17:01:03 -0700 Subject: [PATCH] feat: add reporting as a top level primitive MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a reports system that lets users define TypeScript extensions which analyze model and workflow execution results and produce markdown and JSON output. Reports are user-defined TypeScript files in `extensions/reports/` that implement a `ReportDefinition` interface with a name, description, scope, optional labels, and an async `execute()` function. They run automatically after model method runs or workflow runs, and their output is persisted as versioned data artifacts. Key capabilities: - Three report scopes: method (single method execution), model (model-level aggregation), and workflow (full workflow run with all step results) - Three-level control model for report selection: model-type defaults in the model definition, definition YAML overrides (`reports.require`/`reports.skip` with optional method scoping), and CLI flags (`--report`, `--skip-report`, `--report-label`, `--skip-report-label`, `--skip-reports`) - Report results stored as data artifacts with 30-day lifetime and 5-version retention, tagged with report metadata for queryability - User report loader that discovers, bundles, validates, and caches report extensions with mtime-based invalidation CLI commands: - `swamp report describe ` — show report definition metadata - `swamp report get ` — retrieve a stored report's content - `swamp report search [query]` — interactive fuzzy search across all stored reports with filters for model, workflow, scope, and label Also adds a markdown-to-terminal renderer (via marked + marked-terminal) for rich report output, integrates report extensions into the extension push/pull system, and adds a `swamp-report` skill for Claude-assisted report authoring. --- .claude/skills/swamp-extension-model/SKILL.md | 1 + .claude/skills/swamp-report/SKILL.md | 280 +++++++ .../swamp-report/references/report-types.md | 151 ++++ deno.json | 4 +- deno.lock | 594 +++++--------- design/extension.md | 48 +- design/reports.md | 342 ++++++++ integration/architecture_boundary_test.ts | 2 +- integration/ddd_layer_rules_test.ts | 4 +- src/cli/auto_resolver_adapters.ts | 3 + src/cli/commands/extension_pull.ts | 58 ++ src/cli/commands/extension_push.ts | 55 ++ src/cli/commands/extension_search.ts | 3 + src/cli/commands/extension_update.ts | 3 + src/cli/commands/model_method_run.ts | 26 + src/cli/commands/report.ts | 35 + src/cli/commands/report_describe.ts | 57 ++ src/cli/commands/report_get.ts | 99 +++ src/cli/commands/report_search.ts | 200 +++++ src/cli/commands/workflow_run.ts | 28 + src/cli/mod.ts | 107 +-- src/cli/resolve_extension_files.ts | 34 +- src/cli/resolve_reports_dir.ts | 40 + src/domain/definitions/definition.ts | 19 + src/domain/definitions/definition_test.ts | 1 + .../extension_collective_validator.ts | 12 +- .../extension_collective_validator_test.ts | 1 + src/domain/extensions/extension_content.ts | 12 +- .../extensions/extension_content_extractor.ts | 68 +- .../extension_content_extractor_test.ts | 1 + src/domain/extensions/extension_manifest.ts | 6 +- src/domain/models/model.ts | 26 + src/domain/models/user_model_loader.ts | 2 + src/domain/reports/mod.ts | 41 + src/domain/reports/report.ts | 49 ++ src/domain/reports/report_context.ts | 105 +++ src/domain/reports/report_data_handles.ts | 67 ++ .../reports/report_data_handles_test.ts | 132 ++++ .../reports/report_execution_service.ts | 364 +++++++++ .../reports/report_execution_service_test.ts | 731 ++++++++++++++++++ src/domain/reports/report_registry.ts | 86 +++ src/domain/reports/report_registry_test.ts | 97 +++ src/domain/reports/report_selection.ts | 53 ++ src/domain/reports/user_report_loader.ts | 330 ++++++++ src/domain/workflows/execution_events.ts | 18 + src/domain/workflows/execution_service.ts | 141 ++++ src/domain/workflows/workflow.ts | 18 + src/domain/workflows/workflow_test.ts | 54 ++ src/infrastructure/assets/skill_assets.ts | 8 + src/infrastructure/persistence/paths.ts | 2 + .../persistence/repo_marker_repository.ts | 1 + src/libswamp/mod.ts | 23 + src/libswamp/models/model_method_run_view.ts | 32 + src/libswamp/models/run.ts | 191 ++++- src/libswamp/reports/describe.ts | 57 ++ src/libswamp/reports/describe_test.ts | 87 +++ src/libswamp/reports/get.ts | 277 +++++++ src/libswamp/reports/get_test.ts | 315 ++++++++ src/libswamp/reports/report_views.ts | 71 ++ src/libswamp/reports/search.ts | 201 +++++ src/libswamp/reports/search_test.ts | 274 +++++++ src/libswamp/workflows/run.ts | 291 ++++++- src/libswamp/workflows/run_test.ts | 119 +++ src/libswamp/workflows/workflow_run_view.ts | 3 + src/presentation/markdown_renderer.ts | 48 ++ src/presentation/markdown_renderer_test.ts | 42 + .../output/extension_push_output.ts | 19 + .../output/extension_push_output_test.ts | 2 + .../output/report_search_output.tsx | 277 +++++++ .../renderers/model_method_run.ts | 22 + src/presentation/renderers/report_describe.ts | 69 ++ .../renderers/report_describe_test.ts | 95 +++ src/presentation/renderers/report_get.ts | 75 ++ src/presentation/renderers/report_get_test.ts | 167 ++++ src/presentation/renderers/workflow_run.ts | 22 + 75 files changed, 6932 insertions(+), 466 deletions(-) create mode 100644 .claude/skills/swamp-report/SKILL.md create mode 100644 .claude/skills/swamp-report/references/report-types.md create mode 100644 design/reports.md create mode 100644 src/cli/commands/report.ts create mode 100644 src/cli/commands/report_describe.ts create mode 100644 src/cli/commands/report_get.ts create mode 100644 src/cli/commands/report_search.ts create mode 100644 src/cli/resolve_reports_dir.ts create mode 100644 src/domain/reports/mod.ts create mode 100644 src/domain/reports/report.ts create mode 100644 src/domain/reports/report_context.ts create mode 100644 src/domain/reports/report_data_handles.ts create mode 100644 src/domain/reports/report_data_handles_test.ts create mode 100644 src/domain/reports/report_execution_service.ts create mode 100644 src/domain/reports/report_execution_service_test.ts create mode 100644 src/domain/reports/report_registry.ts create mode 100644 src/domain/reports/report_registry_test.ts create mode 100644 src/domain/reports/report_selection.ts create mode 100644 src/domain/reports/user_report_loader.ts create mode 100644 src/libswamp/reports/describe.ts create mode 100644 src/libswamp/reports/describe_test.ts create mode 100644 src/libswamp/reports/get.ts create mode 100644 src/libswamp/reports/get_test.ts create mode 100644 src/libswamp/reports/report_views.ts create mode 100644 src/libswamp/reports/search.ts create mode 100644 src/libswamp/reports/search_test.ts create mode 100644 src/presentation/markdown_renderer.ts create mode 100644 src/presentation/markdown_renderer_test.ts create mode 100644 src/presentation/output/report_search_output.tsx create mode 100644 src/presentation/renderers/report_describe.ts create mode 100644 src/presentation/renderers/report_describe_test.ts create mode 100644 src/presentation/renderers/report_get.ts create mode 100644 src/presentation/renderers/report_get_test.ts diff --git a/.claude/skills/swamp-extension-model/SKILL.md b/.claude/skills/swamp-extension-model/SKILL.md index b0a29006..dc63b5e3 100644 --- a/.claude/skills/swamp-extension-model/SKILL.md +++ b/.claude/skills/swamp-extension-model/SKILL.md @@ -122,6 +122,7 @@ export const model = { | `inputsSchema` | No | Zod schema for runtime inputs | | `methods` | Yes | Object of method definitions with `arguments` Zod | | `checks` | No | Pre-flight checks run before mutating methods | +| `reports` | No | Inline report definitions (see `swamp-report`) | ## Resources & Files diff --git a/.claude/skills/swamp-report/SKILL.md b/.claude/skills/swamp-report/SKILL.md new file mode 100644 index 00000000..3f6fe964 --- /dev/null +++ b/.claude/skills/swamp-report/SKILL.md @@ -0,0 +1,280 @@ +--- +name: swamp-report +description: Create, register, configure, and run reports for swamp models and workflows. Use when creating report extensions, configuring reports in definition YAML, running reports via CLI, or viewing report output. Triggers on "report", "swamp report", "model report", "create report", "run report", "report extension", "report label", "skip report", "report output", "cost report", "audit report", "workflow report", "report results". +--- + +# Swamp Report Skill + +Create and run reports that analyze model and workflow executions. Reports +produce markdown (human-readable) and JSON (machine-readable) output. All +commands support `--json` for machine-readable output. + +**Verify CLI syntax:** If unsure about exact flags or subcommands, run +`swamp help model report` or `swamp help model method run` for the complete, +up-to-date CLI schema. + +## Quick Reference + +| Task | Command | +| ------------------------ | -------------------------------------------------------------------- | +| Run reports for a model | `swamp model report --json` | +| Filter by label | `swamp model report --label cost --json` | +| Simulate method context | `swamp model report --method create --json` | +| Run method with reports | `swamp model method run --json` | +| Skip all reports | `swamp model method run --skip-reports --json` | +| Skip report by name | `swamp model method run --skip-report -j` | +| Skip report by label | `swamp model method run --skip-report-label -j` | +| Run only named report | `swamp model method run --report -j` | +| Run only labeled reports | `swamp model method run --report-label -j` | +| Workflow with reports | `swamp workflow run --json` | +| Workflow skip reports | `swamp workflow run --skip-reports --json` | + +## Creating a Standalone Report Extension + +Reports are standalone TypeScript files in `extensions/reports/`. Each file +exports a `report` object with a `name`, `description`, `scope`, optional +`labels`, and an `execute` function. + +```typescript +// extensions/reports/cost_report.ts +export const report = { + name: "@myorg/cost-report", + description: "Estimate costs for the executed method", + scope: "method", + labels: ["cost", "finops"], + execute: async (context) => { + const modelName = context.definition.name; + const method = context.methodName; + const status = context.executionStatus; + + return { + markdown: + `# Cost Report\n\n- **Model**: ${modelName}\n- **Method**: ${method}\n- **Status**: ${status}\n`, + json: { modelName, method, status }, + }; + }, +}; +``` + +### Name Conventions + +Report names must follow the `@collective/name` pattern (e.g., +`@myorg/cost-report`). This matches the same collective conventions used by +models, drivers, vaults, and datastores. + +### Report Scopes + +| Scope | Context type | When it runs | +| ---------- | ----------------------- | ------------------------------- | +| `method` | `MethodReportContext` | After a single method execution | +| `model` | `ModelReportContext` | After all method-scope reports | +| `workflow` | `WorkflowReportContext` | After a workflow run completes | + +**Method context** includes: `modelType`, `modelId`, `definition`, `globalArgs`, +`methodName`, `executionStatus`, `dataHandles`. + +**Workflow context** includes: `workflowId`, `workflowRunId`, `workflowName`, +`workflowStatus`, `stepExecutions[]` (each with `jobName`, `stepName`, +`modelName`, `modelType`, `methodName`, `status`, `dataHandles`). + +All contexts include: `repoDir`, `logger`, `dataRepository`, +`definitionRepository`. + +Reports are generic — they receive a `ReportContext` and decide at runtime how +to handle their inputs. They don't declare which model types they support. + +See [references/report-types.md](references/report-types.md) for full type +definitions. + +### Key Rules + +1. **Return both markdown and json** — every report must produce both +2. **Labels are optional** — use them for filtering (e.g., `["cost", "audit"]`) +3. **One report per file** — export a single `report` object from each file +4. **Use scope correctly** — method-scope for per-execution analysis, + model-scope for cross-method analysis, workflow-scope for multi-step + aggregation + +## Three-Level Report Control Model + +Reports are controlled at three levels, from most general to most specific: + +### 1. Model Type Defaults (TypeScript `ModelDefinition`) + +The `reports` field on model definitions lists standalone report names that are +defaults for any model of this type: + +```typescript +// extensions/models/my_model.ts +export const model = { + type: "@myorg/ec2", + version: "2026.03.01.1", + reports: ["@myorg/cost-report", "@myorg/drift-report"], + // ... methods, resources, etc. +}; +``` + +### 2. Definition YAML Overrides (`reportSelection`) + +The `reports:` field in definition YAML provides per-definition overrides. +`require` adds reports beyond model-type defaults. `skip` removes reports from +the defaults. + +```yaml +# definitions/my-vpc.yaml +id: 550e8400-e29b-41d4-a716-446655440000 +name: my-vpc +version: 1 +tags: {} +reports: + require: + - "@myorg/compliance-report" # adds to model-type defaults + - name: security-audit # only run for these methods + methods: ["create", "delete"] + skip: + - "@myorg/drift-report" # removes from model-type defaults +globalArguments: + cidrBlock: "10.0.0.0/16" +methods: + create: + arguments: {} +``` + +### 3. Workflow YAML Overrides + +The `reports:` field in workflow YAML controls workflow-scope reports and can +also override model-level reports for the workflow run. + +```yaml +# workflows/deploy.yaml +name: deploy +reports: + require: + - "@myorg/workflow-summary" # workflow-scope report + skip: + - "@myorg/cost-report" # skip for all models in this workflow +``` + +### Filtering Semantics + +For **method/model scope** reports, the candidate set is: + +- Model-type defaults (`ModelDefinition.reports`) +- Plus definition YAML `require` +- Minus definition YAML `skip` (always wins) +- Minus CLI skip flags (unless report is in `require`) +- Narrowed by CLI inclusion flags (`--report`, `--report-label`) + +For **workflow scope** reports, the candidate set is: + +- Workflow YAML `require` (no model-type defaults apply) +- Minus workflow YAML `skip` +- Minus CLI skip flags (unless in `require`) +- Narrowed by CLI inclusion flags + +### Precedence Rules + +- `skip` always wins — even over `require` for the same report name +- `require` makes reports immune to `--skip-reports`, `--skip-report `, + and `--skip-report-label