|
| 1 | +# OPCODE — Architecture |
| 2 | + |
| 3 | +## Directory Structure |
| 4 | + |
| 5 | +```plaintext |
| 6 | +opcode/ |
| 7 | +├── cmd/opcode/ # Entry point, DI wiring, SSE daemon + panel |
| 8 | +│ └── main.go # Assembles all components, HTTP server (MCP SSE + panel) |
| 9 | +├── internal/ |
| 10 | +│ ├── actions/ # Action interface and built-in action implementations |
| 11 | +│ │ ├── action.go # Action, ActionRegistry, ActionInput/Output interfaces |
| 12 | +│ │ ├── registry.go # Thread-safe Registry with plugin namespace support |
| 13 | +│ │ ├── builtin.go # RegisterBuiltins wiring function |
| 14 | +│ │ ├── http.go # HTTP actions (http.request, http.get, http.post) |
| 15 | +│ │ ├── fs.go # Filesystem actions (fs.read, fs.write, fs.list, fs.stat, fs.delete) |
| 16 | +│ │ ├── shell.go # Shell action (shell.exec) with isolator integration |
| 17 | +│ │ ├── crypto.go # Crypto actions (crypto.hash, crypto.hmac, crypto.uuid, crypto.encode, crypto.decode) |
| 18 | +│ │ ├── assert.go # Assertion actions (assert.equal, assert.schema, assert.truthy) |
| 19 | +│ │ └── workflow.go # Workflow actions (workflow.run, workflow.emit, workflow.context, workflow.fail, workflow.log, workflow.notify) |
| 20 | +│ ├── engine/ # Core execution engine |
| 21 | +│ │ ├── executor.go # Executor interface + implementation (Run, Resume, Signal, Extend, Cancel, Status) |
| 22 | +│ │ ├── executor_flow.go # Flow control step execution (condition, loop, parallel, wait) |
| 23 | +│ │ ├── dag.go # DAG parser with Kahn's topological sort and level computation |
| 24 | +│ │ ├── fsm.go # Workflow FSM + Step FSM with transition validation and event emission |
| 25 | +│ │ ├── worker.go # Bounded goroutine pool with backpressure and panic recovery |
| 26 | +│ │ ├── retry.go # Retry logic with none/linear/exponential/constant backoff |
| 27 | +│ │ ├── circuit_breaker.go # Per-action circuit breaker (closed/open/half-open) |
| 28 | +│ │ └── error_handler.go # On-error strategy dispatch (ignore, fail_workflow, fallback_step) |
| 29 | +│ ├── expressions/ # Expression evaluation and interpolation |
| 30 | +│ │ ├── engine.go # Engine interface (CEL, GoJQ, Expr) |
| 31 | +│ │ ├── cel.go # CEL engine for condition evaluation |
| 32 | +│ │ ├── gojq.go # GoJQ engine for JSON transforms |
| 33 | +│ │ ├── expr.go # Expr engine for logic expressions |
| 34 | +│ │ ├── interpolation.go # ${{...}} two-pass interpolation (variables then secrets) |
| 35 | +│ │ └── scope.go # InterpolationScope builder |
| 36 | +│ ├── diagram/ # Workflow visualization |
| 37 | +│ │ ├── model.go # DiagramModel intermediate representation (nodes, edges, subgraphs) |
| 38 | +│ │ ├── builder.go # WorkflowDefinition + StepStates → DiagramModel |
| 39 | +│ │ ├── ascii.go # ASCII renderer (box-drawing characters, status tags) |
| 40 | +│ │ ├── mermaid.go # Mermaid flowchart renderer (classDef status colors) |
| 41 | +│ │ └── graphviz.go # PNG renderer via go-graphviz (DOT graph → image) |
| 42 | +│ ├── panel/ # Web management panel |
| 43 | +│ │ ├── server.go # PanelServer, HTTP handler, template parsing, embed.FS |
| 44 | +│ │ ├── handlers.go # Page handlers (dashboard, workflows, templates, decisions, etc.) |
| 45 | +│ │ ├── api.go # POST/DELETE endpoints (resolve decision, cancel workflow, etc.) |
| 46 | +│ │ ├── sse.go # EventHub → SSE bridge (global + per-workflow streams) |
| 47 | +│ │ ├── templates/ # Go html/templates (base layout + per-page) |
| 48 | +│ │ └── static/ # Embedded assets (pico.css, htmx.js, mermaid.js, panel.css) |
| 49 | +│ ├── store/ # Persistence layer |
| 50 | +│ │ ├── store.go # Store interface (~20 methods) |
| 51 | +│ │ ├── types.go # Domain types (Workflow, Event, StepState, PendingDecision, etc.) |
| 52 | +│ │ ├── libsql.go # libSQL implementation |
| 53 | +│ │ ├── eventlog.go # EventLog wrapper with replay capability |
| 54 | +│ │ ├── migrations.go # Migration runner |
| 55 | +│ │ └── migrations/ # SQL migration files |
| 56 | +│ │ └── 001_initial_schema.sql |
| 57 | +│ ├── reasoning/ # Reasoning node support |
| 58 | +│ │ ├── context.go # BuildDecisionContext (collects step outputs, intent, data injects) |
| 59 | +│ │ └── validation.go # ValidateResolution (choice against available options) |
| 60 | +│ ├── identity/ # Agent identity management |
| 61 | +│ │ └── agent.go # Agent registration and validation |
| 62 | +│ ├── secrets/ # Secret vault |
| 63 | +│ │ ├── vault.go # Vault interface (Resolve, Store, Delete, List) |
| 64 | +│ │ └── aes_vault.go # AES-256-GCM implementation with PBKDF2 key derivation |
| 65 | +│ ├── isolation/ # Process isolation |
| 66 | +│ │ ├── isolator.go # Isolator interface, ResourceLimits, path validation |
| 67 | +│ │ ├── linux.go # LinuxIsolator (cgroups v2, PID namespace, memory/CPU limits) |
| 68 | +│ │ ├── fallback.go # FallbackIsolator (os/exec + timeout, path validation only) |
| 69 | +│ │ ├── factory_linux.go # Auto-detect: Linux → LinuxIsolator |
| 70 | +│ │ └── factory_default.go # Auto-detect: non-Linux → FallbackIsolator |
| 71 | +│ ├── plugins/ # MCP plugin system |
| 72 | +│ │ ├── manager.go # PluginManager (lifecycle, health checks, restart, action discovery) |
| 73 | +│ │ └── provider.go # Plugin provider interface |
| 74 | +│ ├── streaming/ # Real-time event pub/sub |
| 75 | +│ │ ├── hub.go # EventHub interface |
| 76 | +│ │ └── memory_hub.go # In-memory fan-out implementation |
| 77 | +│ ├── scheduler/ # Cron-based job scheduling |
| 78 | +│ │ └── scheduler.go # Background loop, missed-run recovery, dedup |
| 79 | +│ ├── validation/ # Workflow definition validation |
| 80 | +│ │ ├── validator.go # Validator interface |
| 81 | +│ │ ├── workflow.go # Full workflow validation pipeline |
| 82 | +│ │ ├── dag_check.go # DAG-specific checks (cycles, orphans, missing deps) |
| 83 | +│ │ ├── semantic.go # Semantic validation (action existence, param types) |
| 84 | +│ │ └── jsonschema.go # JSON Schema Draft 2020-12 validation |
| 85 | +│ └── logging/ # Structured logging with correlation IDs |
| 86 | +│ └── context.go # Context-based workflow/step/agent ID propagation |
| 87 | +├── pkg/ |
| 88 | +│ ├── schema/ # Public API types |
| 89 | +│ │ ├── workflow.go # WorkflowDefinition, StepDefinition, configs (Reasoning, Parallel, Loop, Condition, Wait) |
| 90 | +│ │ ├── events.go # Event type constants, WorkflowStatus, StepStatus |
| 91 | +│ │ ├── errors.go # OpcodeError with typed error codes and retryability |
| 92 | +│ │ ├── signal.go # Signal types, DAGMutation, VariableSet |
| 93 | +│ │ └── validation.go # ValidationResult, ValidationIssue |
| 94 | +│ └── mcp/ # MCP server and tool definitions |
| 95 | +│ ├── server.go # OpcodeServer (6 tools, SSE transport) |
| 96 | +│ └── tools.go # Tool handlers (run, status, signal, define, query, diagram) |
| 97 | +├── tests/ |
| 98 | +│ └── e2e/ # End-to-end integration tests |
| 99 | +├── scripts/ |
| 100 | +│ └── test-linux.sh # Docker-based Linux isolation test runner |
| 101 | +├── Dockerfile.test # Docker image for Linux cgroup tests |
| 102 | +├── go.mod |
| 103 | +└── go.sum |
| 104 | +``` |
| 105 | + |
| 106 | +--- |
| 107 | + |
| 108 | +## Request Lifecycle |
| 109 | + |
| 110 | +```mermaid |
| 111 | +sequenceDiagram |
| 112 | + participant Agent as MCP Client (Agent) |
| 113 | + participant MCP as MCP Server |
| 114 | + participant Exec as Executor |
| 115 | + participant DAG as DAG Parser |
| 116 | + participant Pool as WorkerPool |
| 117 | + participant Action as Action |
| 118 | + participant Store as Store (libSQL) |
| 119 | +
|
| 120 | + Agent->>MCP: tools/call (opcode.run) |
| 121 | + MCP->>Store: Resolve template + Create workflow |
| 122 | + MCP->>Exec: Run(workflow, params) |
| 123 | + Exec->>DAG: ParseDAG(steps) |
| 124 | + DAG-->>Exec: Levels (topological sort) |
| 125 | +
|
| 126 | + loop Each DAG level |
| 127 | + Exec->>Pool: Dispatch steps in parallel |
| 128 | + Pool->>Action: Interpolate ${{}} + Execute |
| 129 | + Action-->>Pool: ActionOutput |
| 130 | + Pool->>Store: Append events + Upsert step_state |
| 131 | + end |
| 132 | +
|
| 133 | + Exec-->>MCP: ExecutionResult |
| 134 | + MCP-->>Agent: JSON-RPC response |
| 135 | +``` |
| 136 | + |
| 137 | +**Key behaviors during execution:** |
| 138 | + |
| 139 | +- Each step transitions through FSM states: `pending -> scheduled -> running -> completed/failed` |
| 140 | +- Reasoning steps create a PendingDecision, emit `decision_requested`, and suspend the workflow |
| 141 | +- On failure, the error handler applies retry policies (with backoff) or on_error strategies |
| 142 | +- All state changes are persisted as append-only events; `step_state` is a materialized view |
| 143 | + |
| 144 | +--- |
| 145 | + |
| 146 | +## Event Sourcing Model |
| 147 | + |
| 148 | +OPCODE uses event sourcing as its persistence strategy. Every state change is recorded as an immutable event. The `step_state` table is a materialized view derived from events. |
| 149 | + |
| 150 | +```mermaid |
| 151 | +sequenceDiagram |
| 152 | + participant E as Executor |
| 153 | + participant DB as Events Table (Append-only) |
| 154 | + participant MV as Step State (Materialized View) |
| 155 | + participant R as Resume Logic |
| 156 | +
|
| 157 | + Note over E, DB: Write Path |
| 158 | + E->>DB: append(Workflow_Started) |
| 159 | + E->>DB: append(Step_Started) |
| 160 | + DB-->>MV: Update view (Async) |
| 161 | + E->>DB: append(Step_Completed) |
| 162 | +
|
| 163 | + Note over R, E: Read Path (Resume) |
| 164 | + R->>DB: query(all events for ID) |
| 165 | + DB-->>R: list of events |
| 166 | + R->>R: Replay Events (Reconstruct State) |
| 167 | + R->>E: Continue Execution from last Step |
| 168 | +``` |
| 169 | + |
| 170 | +This design enables: |
| 171 | + |
| 172 | +- **Full auditability** -- every action is recorded with a timestamp and sequence number |
| 173 | +- **Reliable resume** -- after a crash or suspension, events are replayed to rebuild exact state |
| 174 | +- **Reasoning node safety** -- decisions stored as `decision_resolved` events, never replayed (the agent's choice is final) |
| 175 | + |
| 176 | +--- |
| 177 | + |
| 178 | +## State Machines |
| 179 | + |
| 180 | +**Workflow FSM:** |
| 181 | + |
| 182 | +```mermaid |
| 183 | +stateDiagram-v2 |
| 184 | + [*] --> pending |
| 185 | + pending --> active |
| 186 | + active --> completed |
| 187 | + active --> failed |
| 188 | + active --> cancelled |
| 189 | + active --> suspended |
| 190 | + suspended --> active: resume |
| 191 | + suspended --> cancelled |
| 192 | + suspended --> failed |
| 193 | +``` |
| 194 | + |
| 195 | +**Step FSM:** |
| 196 | + |
| 197 | +```mermaid |
| 198 | +stateDiagram-v2 |
| 199 | + [*] --> pending |
| 200 | + pending --> scheduled |
| 201 | + scheduled --> running |
| 202 | + running --> completed |
| 203 | + running --> failed |
| 204 | + running --> suspended |
| 205 | + running --> retrying |
| 206 | + running --> skipped |
| 207 | + suspended --> running: resume |
| 208 | + suspended --> failed |
| 209 | + suspended --> skipped |
| 210 | + retrying --> running: retry |
| 211 | + retrying --> failed |
| 212 | +``` |
| 213 | + |
| 214 | +--- |
| 215 | + |
| 216 | +## Database Schema |
| 217 | + |
| 218 | +The embedded libSQL database contains 10 tables: |
| 219 | + |
| 220 | +| Table | Purpose | |
| 221 | +| -------------------- | ------------------------------------------------------------- | |
| 222 | +| `agents` | Registered agent identities (LLM, system, human, service) | |
| 223 | +| `workflows` | Workflow execution records with status, definition, I/O | |
| 224 | +| `events` | Append-only event sourcing log (unique per workflow+sequence) | |
| 225 | +| `step_state` | Materialized view of current step execution state | |
| 226 | +| `workflow_context` | Per-workflow metadata (intent, agent notes, accumulated data) | |
| 227 | +| `pending_decisions` | Reasoning nodes awaiting agent input | |
| 228 | +| `workflow_templates` | Reusable workflow definitions (name+version composite key) | |
| 229 | +| `plugins` | Registered MCP plugin subprocesses | |
| 230 | +| `secrets` | AES-256-GCM encrypted key-value secrets | |
| 231 | +| `scheduled_jobs` | Cron-triggered workflow executions | |
| 232 | + |
| 233 | +An additional `audit_log` table is defined for flag-activated agent action auditing. |
| 234 | + |
| 235 | +--- |
| 236 | + |
| 237 | +## ASCII Diagram Format |
| 238 | + |
| 239 | +Text-based box-drawing diagram for CLI agents and terminal output. When [`mermaid-ascii`](https://github.com/AlexanderGrooff/mermaid-ascii) is installed (auto-downloaded by `opcode install`), diagrams use its layout engine for superior branching, edge routing, and diamond merges. Falls back to a built-in renderer otherwise. |
| 240 | + |
| 241 | +```plaintext |
| 242 | +=== Workflow === |
| 243 | +
|
| 244 | +┌───────┐ |
| 245 | +│ Start │ |
| 246 | +└───────┘ |
| 247 | + │ |
| 248 | + ▼ |
| 249 | +┌────────────┐ |
| 250 | +│ fetch-data │ |
| 251 | +│ [OK] │ |
| 252 | +│ 450ms │ |
| 253 | +└────────────┘ |
| 254 | + │ |
| 255 | + ▼ |
| 256 | +┌──────────┐ |
| 257 | +│ validate │ |
| 258 | +│ [OK] │ |
| 259 | +│ 12ms │ |
| 260 | +└──────────┘ |
| 261 | + │ |
| 262 | + ▼ |
| 263 | +┌────────┐ |
| 264 | +│ decide │ |
| 265 | +│ [WAIT] │ |
| 266 | +└────────┘ |
| 267 | + │ |
| 268 | + ▼ |
| 269 | +┌─────────┐ |
| 270 | +│ process │ |
| 271 | +│ [PEND] │ |
| 272 | +└─────────┘ |
| 273 | + │ |
| 274 | + ▼ |
| 275 | +┌────────┐ |
| 276 | +│ notify │ |
| 277 | +│ [PEND] │ |
| 278 | +└────────┘ |
| 279 | + │ |
| 280 | + ▼ |
| 281 | +┌─────┐ |
| 282 | +│ End │ |
| 283 | +└─────┘ |
| 284 | +``` |
| 285 | + |
| 286 | +Status tags: `[OK]` completed, `[FAIL]` failed, `[RUN]` running, `[WAIT]` suspended, `[PEND]` pending, `[SKIP]` skipped, `[RETRY]` retrying. |
| 287 | + |
| 288 | +--- |
| 289 | + |
| 290 | +See also: [Design Rationale & Trade-offs](design-rationale.md) | [Benchmarks](benchmarks.md) |
0 commit comments