diff --git a/.github/workflows/promotion.yml b/.github/workflows/promotion.yml index 343295d..03d1f88 100644 --- a/.github/workflows/promotion.yml +++ b/.github/workflows/promotion.yml @@ -6,6 +6,8 @@ on: paths: - promotion.yml - resources/** + # Shared blocks: an edit changes resources without changing their files. + - templates/** workflow_dispatch: inputs: pipeline: diff --git a/AGENTS.md b/AGENTS.md index 1a4c96d..2e0b07c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -39,6 +39,7 @@ change, before running: | --- | --- | | `npm run apply`, `npm run push` | Changes live assistants | | `npm run promote -- … --apply` | Changes the next org (often production) | +| `npm run sync-blocks -- --apply` | Changes live resources in every org that uses the edited block | | `npm run rollback -- --to …` | Reverts live resources | | `npm run cleanup -- --force --confirm ` | Deletes platform resources | | `npm run pull -- --force` | Overwrites local edits | @@ -47,6 +48,7 @@ change, before running: Safe to run without asking: `npm run validate`, `npm run audit`, `npm run check -- … --dry-run`, `npm run promote` without `--apply` (a plan), +`npm run sync-blocks` without `--apply` (a list), plain `npm run pull` (it never overwrites local edits), `npm run push -- --dry-run`, `npm run build`, `npm test`. @@ -182,6 +184,8 @@ how sync, conflicts and output icons work: | Edit an assistant's system prompt | Edit the Markdown body of `resources//assistants/.md` | | Change assistant settings | Edit the YAML frontmatter of the same file | | Add a tool / assistant / squad | Create `resources//tools/.yml`, `assistants/.md`, `squads/.yml` | +| Share prompt text or config between files | Put it in `templates/blocks/.md` or `templates/config/.yml` and include it (see [Blocks](#blocks)) | +| See which resources a block edit reaches | `npm run sync-blocks` (a list; `--apply` deploys and needs a yes) | | Add post-call analysis | Create `resources//structuredOutputs/.yml` and list it in the assistant's `artifactPlan.structuredOutputIds` | | Write simulation tests | Create files under `resources//simulations/` (see [Simulations](#simulations-and-pr-checks)) | | Check files offline | `npm run validate -- ` | @@ -239,6 +243,24 @@ The engine resolves IDs and credential names to each org's UUIDs on push. | The display name | Change `name` in the file (or in the dashboard, then pull). The filename stays. | | The file itself | A renamed file has no state entry, so the next deploy stops at the new-file check. Either keep the old filename, or (with the human's agreement) deploy it as new with `--allow-new-files` and delete the old platform resource with `npm run cleanup -- --force --confirm `. | +### Blocks + +A line that says only `@include ` in a system prompt, or a +`$include: ` key in a mapping, pulls in a shared file from +`resources//templates/` (the org's own) or `templates/` (shared by +every org). The engine expands it before validate, push and PR checks. + +- To change shared wording or settings, edit the block, not the files that + include it. Never replace a marker with the block's text. +- A block edit changes every resource that includes it, in every org. Run + `npm run sync-blocks` and show the human the list before any deploy. +- Don't use Liquid `{% include %}` / `{% render %}` or YAML merge keys (`<<`) + to share content: Vapi and the engine drop them silently. +- Commit `resources//.vapi-blocks.json` when push changes it; never + edit or delete it. + +Syntax, merge rules and dashboard-edit handling: [Blocks](docs/guides/blocks.md). + ### Excluding resources (`.vapi-ignore`) `resources//.vapi-ignore` lists platform resources this repo must not @@ -345,6 +367,7 @@ For changes under `src/`, `tests/` or `.github/`: | --- | --- | | Every resource setting, with examples | [Resource reference](docs/guides/resource-reference.md) | | Minimal tested files | [File formats](docs/guides/file-formats.md), [`examples/starter/`](examples/starter/README.md) | +| Shared prompt text and config | [Blocks](docs/guides/blocks.md), [`examples/blocks/`](examples/blocks/README.md) | | System prompts | [Writing system prompts](docs/guides/writing-prompts.md), [Vapi Prompt Optimization Guide](docs/Vapi%20Prompt%20Optimization%20Guide.md) | | Commands, flags, test-call output | [Commands](docs/guides/commands.md) | | Sync, conflicts, drift, output icons | [How the engine works](docs/guides/how-it-works.md), [sync behavior](docs/learnings/sync-behavior.md) | diff --git a/README.md b/README.md index 9984b3d..e87ee9f 100644 --- a/README.md +++ b/README.md @@ -217,6 +217,7 @@ More detail: [How the engine works](docs/guides/how-it-works.md). | `npm run sim -- ` | Run a simulation suite against deployed resources | | `npm run check` | Run PR simulation checks against local files | | `npm run promote` | Promote resources from one org to the next | +| `npm run sync-blocks` | List, and optionally deploy, the resources a shared block edit reaches | `setup`, `apply`, `pull`, `push`, `cleanup` and `call` prompt interactively when run without arguments. Full flags and examples: @@ -228,6 +229,7 @@ when run without arguments. Full flags and examples: | --- | --- | | [Everyday workflows](docs/guides/workflows.md) | Deploy, pull safely, recover from a bad deploy, clean up | | [File formats](docs/guides/file-formats.md) | Write assistants, tools, squads, structured outputs and simulations | +| [Blocks](docs/guides/blocks.md) | Share prompt text and config between assistants, squads and orgs | | [Resource reference](docs/guides/resource-reference.md) | Look up every setting, with examples | | [Writing system prompts](docs/guides/writing-prompts.md) | Structure a voice agent's prompt | | [PR checks](docs/guides/pr-checks.md) | Test every pull request with simulations | diff --git a/docs/guides/blocks.md b/docs/guides/blocks.md new file mode 100644 index 0000000..10d36a9 --- /dev/null +++ b/docs/guides/blocks.md @@ -0,0 +1,316 @@ +# Blocks: shared prompt text and config + +A block is a piece of prompt text or config that lives in one file and is +used by many resources. Change the block and every assistant and squad that +includes it changes with it, in every org. + +There are two kinds: + +| Kind | File | Included with | Use it for | +| --- | --- | --- | --- | +| Prompt block | `templates/blocks/.md` | a line that says `@include ` in a system prompt | Wording several assistants share: safety rules, tone, a closing script | +| Config block | `templates/config/.yml` | a `$include: ` key in any mapping | Settings several resources share: voice, transcriber, model settings, squad overrides | + +The engine expands blocks when it loads a resource file, before `validate`, +`apply`, `push` and PR checks. Vapi only ever receives the finished resource: +blocks do not exist on the platform or in the dashboard. + +Every snippet below is a file from [`examples/blocks/`](../../examples/blocks/), +two clinics that share a red-flags block and a voice. CI checks that the +snippets match the files and that the example passes `validate`. + +## Prompt blocks + +Put the shared text in a Markdown file: + +```markdown + +# Emergencies + +If the caller mentions chest pain, difficulty breathing or signs of a stroke, +stop what you are doing and tell them to hang up and call 999 now. Then offer +to connect them to {{practice_name}} reception. +``` + +Then write `@include red-flags` on a line of its own wherever the text +belongs in a system prompt: + +```markdown + +--- +name: Triage +firstMessage: Thanks for calling {{practice_name}}. What can I help you with today? +model: + provider: openai + model: gpt-4.1 + temperature: 0.3 +voice: + $include: voice +--- + +# Identity + +You are the triage assistant for {{practice_name}}. You find out what the +caller needs and pass them to the right assistant. + +@include red-flags + +# Flow + +1. Ask what the caller needs help with today. +2. For a repeat prescription, pass them to the Pharmacy assistant. +3. For anything else, take a short message for the reception team. +``` + +- The marker must be the only thing on its line. `@include` in the middle of + a sentence is left as ordinary text. +- It works in every system prompt: the Markdown body of an assistant, a + `role: system` message in a `.yml` assistant, and a system message inside + a squad's `assistantOverrides` or `membersOverrides`. Other text fields, + such as `firstMessage`, are not expanded. +- The name is the file's path under `templates/blocks/` without `.md`, so + `templates/blocks/safety/red-flags.md` is `@include safety/red-flags`. +- A block can include other blocks. A block that ends up including itself is + an error. +- A block has no parameters. What differs per call belongs in a Liquid + variable (next section); what differs per org belongs in an org's own + block (see [Lookup order](#lookup-order-shared-blocks-and-an-orgs-own)). + +### Liquid variables + +Nothing inside a block is interpreted. `{{practice_name}}`, +`{% if … %}` and every other Liquid expression reach Vapi exactly as written, +and Vapi fills them on each call, the same as if you had typed the block's +text into the prompt. + +The marker is deliberately not Liquid. Vapi's call-time Liquid has no file +access: it renders `{% include %}`, `{% render %}` and `{% layout %}` as +nothing, so a prompt that relied on them would silently lose that text on a +live call. `validate` warns when a system prompt contains one of those tags. + +## Config blocks + +A config block is a YAML file: + +```yaml +# examples/blocks/templates/config/voice.yml +provider: 11labs +voiceId: sarah +``` + +Any mapping can pull it in with a `$include` key. In the assistant above, +`voice:` with `$include: voice` becomes the two fields of that file. A squad +can do the same, and a block can include another block: + +```yaml +# examples/blocks/templates/config/call-settings.yml +silenceTimeoutSeconds: 20 +maxDurationSeconds: 900 +voice: + $include: voice +``` + +```yaml +# examples/blocks/resources/north-clinic/squads/front-desk.yml +name: Front Desk +members: + - assistantId: triage + - assistantId: pharmacy +membersOverrides: + $include: call-settings + variableValues: + practice_name: North Clinic +``` + +### How a block and the file's own keys combine + +The mapping that holds `$include` can set keys of its own. The rule is +always: **the file wins**. + +| The block has | The file sets, beside `$include` | Result | +| --- | --- | --- | +| a key | nothing for that key | the block's value | +| a key | the same key, a plain value | the file's value | +| a mapping | the same key, a mapping | merged key by key, by these same rules | +| a list | the same key, a list | the file's list replaces the block's list whole. Lists are never joined. | +| anything | the same key, a different kind of value | the file's value | + +So, with the voice block above: + +```yaml +voice: + $include: voice + voiceId: burt # provider comes from the block, voiceId from here +``` + +`$include` takes one name or a list of names: + +```yaml +membersOverrides: + $include: [call-settings, transcriber] +``` + +A block whose content is a list (or a single value) can be included only on +its own, as the whole value of a field: + +```yaml +model: + toolIds: + $include: standard-tools # templates/config/standard-tools.yml is a list +``` + +### What is refused + +These fail at load, so `validate`, `apply` and PR checks all stop with a +message naming the file: + +| Case | Why | +| --- | --- | +| A block that does not exist, in the org's folder or the shared one | A missing block must never become empty text or missing settings. The message lists both places it looked. | +| Blocks that include each other in a loop | No finished text exists. The message shows the chain. | +| Two blocks in one `$include` list that set the same field to different values | The result would depend on the order of the list. Set the field in one block, or in the file. | +| A list block included beside other keys or other blocks | There is no sensible way to merge a list with keys. | +| `$include` in the same mapping as a YAML merge key (`<<`) | Two merge mechanisms in one place. The engine does not process `<<` at all (see below). | +| `$include` with an empty value, a non-text value, or the same name twice | Almost certainly a typo. | + +### YAML anchors and merge keys + +Don't use them for sharing. An anchor only reaches inside one file, and +`pull` writes files back with every alias expanded. A merge key +(`<<: *defaults`) is not processed by the engine at all: it is sent to Vapi +as a field named `<<`. `validate` warns when it finds one. + +## Lookup order: shared blocks and an org's own + +A name is looked up in two places, in this order: + +1. `resources//templates/`, the org's own blocks +2. `templates/` at the repository root, shared by every org + +So an org replaces a shared block by having a file of the same name. An +org's block that includes its own name gets the shared one, which lets it +add to the shared block rather than copy it: + +```markdown + +@include red-flags + +{{practice_name}} has a walk-in centre next door, open until 8pm. If the +caller is not sure it is an emergency, tell them they can go there. +``` + +Config blocks work the same way. This org file keeps everything in the +shared `voice` block and changes one field: + +```yaml +# resources/south-clinic/templates/config/voice.yml +$include: voice +voiceId: burt +``` + +## Rolling a block change out + +A block edit changes what many resources send to Vapi, but it changes none +of their files. Three things follow from that. + +**1. See what it reaches.** + +```bash +npm run sync-blocks # every org; reads files, changes nothing +npm run sync-blocks -- --org acme # one org +``` + +It lists every resource whose blocks have changed since that resource was +last pushed, with the block that changed. An org with its own version of a +block is not listed for an edit to the shared version. + +**2. Push it.** + +```bash +npm run sync-blocks -- --apply +``` + +For each org this runs `npm run apply -- `, so it +needs each org's API key, like any deploy, and it goes through the same +validate → pull → push path. A resource that someone edited in the dashboard +is refused by the drift check, exactly as with `apply`; the command says +which resources were left behind and exits 1. Resolve the edit (see +[Dashboard edits](#dashboard-edits-to-a-file-that-uses-blocks)) and run it +again. + +Plain `npm run apply -- ` also pushes them: to the engine a block edit +looks like a local edit to every file that includes the block. + +**3. Commit the record.** The engine knows what each resource was last pushed +with from `resources//.vapi-blocks.json`. Push updates it; commit it +with your other changes, as you do the state file. Don't edit it by hand. If +it is missing, every resource that uses a block is listed until it is pushed +once. + +### With promotion + +When orgs are linked by a [promotion](promotion.md) pipeline (for example +one org per site, or dev → production), promotion carries block edits to the +orgs it deploys: + +- A shared block is one file for every org, so there is nothing to copy. The + plan shows each destination resource that uses the changed block as + `sync`, and `--apply` pushes it. +- An org's own blocks are not copied to the next org. Each org is expanded + with its own blocks, then the shared ones. +- The promotion workflow runs when `templates/**` changes, as it does for + `resources/**`. +- Promotion deploys destination orgs only. The first org of a pipeline, and + any org outside a pipeline, gets a block edit from `sync-blocks` or + `apply`. + +Promotion pushes with `--resolve=ours`, as it does for every file it +promotes: a dashboard edit in a destination org is overwritten, not refused. + +### With PR checks + +A change under `templates/` selects every [PR check](pr-checks.md), and a +change to an org's own blocks selects that org's checks. The check builds +its payload from the branch's files, so it tests the new block text. + +## Dashboard edits to a file that uses blocks + +The dashboard shows the finished prompt, with the block's text in place of +the marker. If `pull` wrote that back to your file, the marker would be +replaced by a copy of the block and the file would stop following later +block edits. So `pull` never rewrites a file that uses blocks: + +| Situation | An ordinary file | A file that uses blocks | +| --- | --- | --- | +| File and dashboard agree | Rewritten in pull's own format | Left exactly as written | +| Only the dashboard changed | Overwritten with the dashboard version | Kept. The dashboard version is saved beside it, and the edit stays flagged as drift | +| `pull --force` | Overwritten | Kept. The dashboard version is saved beside it | +| Both changed, `--resolve=theirs` | Overwritten | Kept. The dashboard version is saved beside it, and the conflict stays open | + +"Saved beside it" means a `..bkp.md` (or `.yml`) file next +to yours, which git and the engine ignore. To resolve a dashboard edit: + +1. Compare the saved copy with your file and find what was changed. +2. If the change is in text that comes from your file, make the same change + in your file. If it is in text that comes from a block, change the block + (every resource that uses it changes too) or give this org its own block. + To discard the dashboard edit, change nothing. +3. Run `npm run apply -- --resolve=ours `. It pushes your + version over the dashboard's. + +## Limits + +- The drift check compares the dashboard with the last state *your* checkout + saw (see [sync behavior](../learnings/sync-behavior.md)). A fresh clone + has no such baseline, so its first push is not refused. This is the same + for every push, with or without blocks. +- An org with no credentials stored in Vapi gets no drift check at all: + `push` re-syncs such an org's state before every run, which resets the + baseline to whatever the dashboard holds. This also predates blocks (see + `improvements.md`, entry 40). +- Blocks are expanded in `.md`, `.yml` and `.ts` resources. As with any + `.ts` resource, `pull` and drift detection skip `.ts` files. +- A pull request shows a block's diff once. Run `npm run sync-blocks` on the + branch to see every resource and org it reaches. +- `templates/` is not a resource folder. Nothing in it is pushed on its own, + and `cleanup` and `audit` don't look there. diff --git a/docs/guides/commands.md b/docs/guides/commands.md index 299b12f..63e6ce4 100644 --- a/docs/guides/commands.md +++ b/docs/guides/commands.md @@ -21,6 +21,7 @@ The other commands are direct only. | `npm run sim` | `npm run sim -- --suite --target ` | Run a simulation suite against deployed resources. Exits 0 passed, 1 failed, 3 incomplete. | | `npm run check` | `npm run check -- ` or `--all`, `[--dry-run]` | Run [PR checks](pr-checks.md) against local files, nothing deployed. Exits 0 passed, 1 failed, 2 config or build error, 3 incomplete. | | `npm run promote` | `npm run promote -- --pipeline

--from --to [--apply]` | Plan, or apply, a [promotion](promotion.md) between orgs. | +| `npm run sync-blocks` | `npm run sync-blocks [-- --org ] [--apply]` | List the resources, in every org, that a [block](blocks.md) edit has not reached yet. `--apply` deploys them through `apply`. Exits 1 if any is left behind. | | `npm run build` | `npm run build` | Type-check the code and tests. | | `npm test` | `npm test` | Run the test suite. | diff --git a/docs/guides/file-formats.md b/docs/guides/file-formats.md index c8500e9..05c7989 100644 --- a/docs/guides/file-formats.md +++ b/docs/guides/file-formats.md @@ -178,6 +178,15 @@ simulationIds: - books-cleaning-calm ``` +## Blocks (shared text and config) + +Text or settings that several files share can live in one file under +`templates/` and be included: `@include ` on its own line in a system +prompt, or a `$include: ` key in any mapping. An org can replace a +shared block with its own. Syntax, merge rules and how a block change is +rolled out: [Blocks](blocks.md), with a tested example in +[`examples/blocks/`](../../examples/blocks/README.md). + ## TypeScript resources (`.ts`) Any resource can also be a `.ts` file whose default export is the resource diff --git a/docs/guides/pr-checks.md b/docs/guides/pr-checks.md index 58b514f..ef28748 100644 --- a/docs/guides/pr-checks.md +++ b/docs/guides/pr-checks.md @@ -101,6 +101,8 @@ In GitHub → Settings → Secrets and variables → Actions: `{"my-org":""}` (several orgs, or a CI org). - Variable `VAPI_CHECKS_ENABLED=true`. +A check is affected by changes to its org's files, to shared +[blocks](blocks.md) under `templates/`, to the engine, and to its own `paths`. `.github/workflows/vapi-checks.yml` then runs every affected check on each PR push. It asks for `statuses: write` only to post the direct links. diff --git a/docs/guides/promotion.md b/docs/guides/promotion.md index e5b9efc..4f1ceb6 100644 --- a/docs/guides/promotion.md +++ b/docs/guides/promotion.md @@ -67,7 +67,8 @@ manual runs: 5. For a controlled single transition, run the workflow manually and provide `pipeline`, `from`, and `to`. -Automatic runs watch committed changes to `promotion.yml` and `resources/**`. +Automatic runs watch committed changes to `promotion.yml`, `resources/**` and +`templates/**` (shared [blocks](blocks.md#with-promotion)). A manual run with no inputs reconciles every adjacent transition; supplying inputs requires all three values and reconciles only that transition. @@ -81,6 +82,22 @@ For a complete fake dev → staging → production fixture, see the [dummy multi-org example](../../examples/cross-org-promotion/README.md). Nothing under `examples/` is loaded by the engine. +## Blocks + +A resource that includes a [block](blocks.md) can need a push when none of its +files changed, because the block did. The plan lists those destination +resources as `sync`, with the block that changed: + +```text +release: dev → prod + sync assistants/triage.md (block changed: blocks/red-flags.md) +``` + +`--apply` pushes them like any other planned file. Shared blocks under +`templates/` are never copied, since every org reads the same folder, and an +org's own blocks under `resources//templates/` stay in that org. The +plan fails if a promoted file includes a block the destination can't find. + ## Source-org and deletion boundary Promotion treats `resources//` in Git as the reviewed desired state for diff --git a/docs/learnings/squads.md b/docs/learnings/squads.md index 07fb1c6..249cca2 100644 --- a/docs/learnings/squads.md +++ b/docs/learnings/squads.md @@ -165,7 +165,7 @@ The final assistant configuration is built by merging these layers in order: Later layers win on conflicts. `variableValues` from all layers are merged separately. -**Gotcha:** Liquid template substitution replaces undefined variables with **empty strings**, not errors. If a variable isn't in the merged `variableValues`, any `{{ myVar }}` reference silently becomes `""`. +**Gotcha:** a variable with no value is **left in the prompt as written**, not replaced and not an error. If a variable isn't in the merged `variableValues`, `{{ myVar }}` reaches the model as the literal text `{{ myVar }}`. A variable that is set to an empty string does render as nothing. So a squad that forgets a variable shows the braces to the model: check every `{{ … }}` in a prompt, and in any [block](../guides/blocks.md) it includes, has a value in each squad that uses it. --- diff --git a/docs/learnings/sync-behavior.md b/docs/learnings/sync-behavior.md index b7532ad..fda1ff8 100644 --- a/docs/learnings/sync-behavior.md +++ b/docs/learnings/sync-behavior.md @@ -134,6 +134,27 @@ Both `pull` and `push` treat this as clean (live sides agree — nothing to reconcile). The no-op write/PATCH re-seeds the baseline, self-healing the stale pointer. Never blocks, never prompts. +### 6. The local file uses blocks + +A file with an `@include` line or a `$include` key is compared in its expanded +form, so the directions above mean the same thing. What differs is that +`pull` **never writes the file**: the platform holds the expanded resource, +and writing it back would replace each marker with a copy of the block. A +block edit makes every file that includes it `local-ahead`. + +| Direction | `pull` on a file that uses blocks | +| --- | --- | +| L = D (`clean`, or no baseline yet) | 🧩 file left as written; baseline := the shared hash | +| `local-ahead` | ⬆️ preserved, as for any file | +| `dashboard-ahead` | ⚠️ file kept; dashboard version written beside it as `..bkp.`; **baseline untouched**, so the edit is still drift at the next pull and push | +| `both-diverged` | as scenario 4, except `--resolve=theirs` keeps the file, writes the `.bkp.` copy and leaves the conflict open | +| any, with `--force` | ⚠️ file kept; `.bkp.` copy written when the two differ | +| L ≠ D, no baseline | ✏️ file kept, no baseline written | + +To resolve a kept dashboard edit, merge it by hand into the file or the +block, then `npm run apply -- --resolve=ours `. See +[Blocks](../guides/blocks.md#dashboard-edits-to-a-file-that-uses-blocks). + --- ## Existence scenarios (something is missing somewhere) diff --git a/docs/learnings/yaml-conventions.md b/docs/learnings/yaml-conventions.md index 2efc2d5..2ec9fde 100644 --- a/docs/learnings/yaml-conventions.md +++ b/docs/learnings/yaml-conventions.md @@ -143,15 +143,15 @@ Mixing `|` and `|-` inconsistently across resources makes the rendered output ha --- -## Anchors and Aliases: Engine Support Varies +## Anchors, Aliases and Merge Keys: Use Blocks Instead -YAML supports `&anchor` and `*alias` for DRY-ing repeated structures. **Many gitops engines do NOT round-trip anchors faithfully** — they parse them, but on save they expand the alias inline. This means: +YAML supports `&anchor` and `*alias` for DRY-ing repeated structures, and `<<: *alias` for "this mapping plus overrides". Neither is a safe way to share content here: -- A resource file you wrote with anchors will get rewritten with the anchors expanded after the next push. -- The diff after `npm run pull` will look enormous because every alias becomes its full expansion. -- Reviewers reading the rewritten file lose the "this section is intentionally a copy of X" signal. +- **An alias is parsed, but pull writes it back expanded.** `npm run pull` rewrites a resource file from the platform's copy, so every alias becomes its full expansion, the diff looks enormous, and reviewers lose the "this section is intentionally a copy of X" signal. +- **A merge key (`<<`) is not processed at all.** The engine parses it as an ordinary key and sends it to Vapi as a field named `<<`, so the "defaults plus overrides" pattern does not work. `npm run validate` warns when it finds one. +- **An anchor only reaches inside one file**, and never into the Markdown body where an assistant's prompt lives. -**Recommendation:** avoid anchors/aliases in gitops resource YAML even if your local parser supports them. Duplicate the content explicitly. The maintenance cost of duplication is real but visible; the maintenance cost of silently-expanding anchors is invisible until someone tries to update both copies and only finds one. +**Recommendation:** share content with [blocks](../guides/blocks.md). A config block is a YAML file under `templates/config/` that any mapping includes with `$include: `, with the file's own keys winning; a prompt block is a Markdown file under `templates/blocks/` that a system prompt includes with `@include `. Blocks work across files and orgs, and pull never rewrites a file that uses them. --- diff --git a/examples/blocks/README.md b/examples/blocks/README.md new file mode 100644 index 0000000..48c4db3 --- /dev/null +++ b/examples/blocks/README.md @@ -0,0 +1,42 @@ +# Blocks example: two clinics sharing prompt text and config + +A minimal example of [blocks](../../docs/guides/blocks.md): shared files that +many resources include. + +```text +examples/blocks/ +├── templates/ +│ ├── blocks/red-flags.md shared prompt block +│ └── config/ +│ ├── voice.yml shared config block +│ └── call-settings.yml shared config block (includes voice) +└── resources/ + ├── north-clinic/ + │ ├── assistants/triage.md @include red-flags, $include: voice + │ ├── assistants/pharmacy.md @include red-flags, $include: voice + │ └── squads/front-desk.yml $include: call-settings + └── south-clinic/ + ├── (the same three files) + └── templates/blocks/red-flags.md South Clinic's own red-flags block +``` + +What to notice: + +- Both assistants in both orgs include the same `red-flags` block. North + Clinic gets the shared text. South Clinic gets its own block, which + includes the shared one and adds a paragraph. +- `{{practice_name}}` inside the blocks is left alone by the engine. Vapi + fills it on each call from the squad's `variableValues`. +- The squad file sets `variableValues` beside `$include: call-settings`: the + block supplies the shared settings and the file adds its own. + +To try it, copy `templates/` to the root of your repository and one of the +org folders to `resources//`, then run: + +```bash +npm run validate -- +npm run sync-blocks +``` + +Both commands only read files. Deploying is `npm run apply -- `, +which changes a live org. diff --git a/examples/blocks/resources/north-clinic/assistants/pharmacy.md b/examples/blocks/resources/north-clinic/assistants/pharmacy.md new file mode 100644 index 0000000..05785c5 --- /dev/null +++ b/examples/blocks/resources/north-clinic/assistants/pharmacy.md @@ -0,0 +1,21 @@ +--- +name: Pharmacy +model: + provider: openai + model: gpt-4.1 + temperature: 0.3 +voice: + $include: voice +--- + +# Identity + +You are the pharmacy assistant for {{practice_name}}. You take repeat +prescription requests. + +@include red-flags + +# Flow + +1. Ask which medicine the caller needs. +2. Read the request back to them and tell them it takes two working days. diff --git a/examples/blocks/resources/north-clinic/assistants/triage.md b/examples/blocks/resources/north-clinic/assistants/triage.md new file mode 100644 index 0000000..8ca4cbe --- /dev/null +++ b/examples/blocks/resources/north-clinic/assistants/triage.md @@ -0,0 +1,23 @@ +--- +name: Triage +firstMessage: Thanks for calling {{practice_name}}. What can I help you with today? +model: + provider: openai + model: gpt-4.1 + temperature: 0.3 +voice: + $include: voice +--- + +# Identity + +You are the triage assistant for {{practice_name}}. You find out what the +caller needs and pass them to the right assistant. + +@include red-flags + +# Flow + +1. Ask what the caller needs help with today. +2. For a repeat prescription, pass them to the Pharmacy assistant. +3. For anything else, take a short message for the reception team. diff --git a/examples/blocks/resources/north-clinic/squads/front-desk.yml b/examples/blocks/resources/north-clinic/squads/front-desk.yml new file mode 100644 index 0000000..ea9830a --- /dev/null +++ b/examples/blocks/resources/north-clinic/squads/front-desk.yml @@ -0,0 +1,8 @@ +name: Front Desk +members: + - assistantId: triage + - assistantId: pharmacy +membersOverrides: + $include: call-settings + variableValues: + practice_name: North Clinic diff --git a/examples/blocks/resources/south-clinic/assistants/pharmacy.md b/examples/blocks/resources/south-clinic/assistants/pharmacy.md new file mode 100644 index 0000000..05785c5 --- /dev/null +++ b/examples/blocks/resources/south-clinic/assistants/pharmacy.md @@ -0,0 +1,21 @@ +--- +name: Pharmacy +model: + provider: openai + model: gpt-4.1 + temperature: 0.3 +voice: + $include: voice +--- + +# Identity + +You are the pharmacy assistant for {{practice_name}}. You take repeat +prescription requests. + +@include red-flags + +# Flow + +1. Ask which medicine the caller needs. +2. Read the request back to them and tell them it takes two working days. diff --git a/examples/blocks/resources/south-clinic/assistants/triage.md b/examples/blocks/resources/south-clinic/assistants/triage.md new file mode 100644 index 0000000..8ca4cbe --- /dev/null +++ b/examples/blocks/resources/south-clinic/assistants/triage.md @@ -0,0 +1,23 @@ +--- +name: Triage +firstMessage: Thanks for calling {{practice_name}}. What can I help you with today? +model: + provider: openai + model: gpt-4.1 + temperature: 0.3 +voice: + $include: voice +--- + +# Identity + +You are the triage assistant for {{practice_name}}. You find out what the +caller needs and pass them to the right assistant. + +@include red-flags + +# Flow + +1. Ask what the caller needs help with today. +2. For a repeat prescription, pass them to the Pharmacy assistant. +3. For anything else, take a short message for the reception team. diff --git a/examples/blocks/resources/south-clinic/squads/front-desk.yml b/examples/blocks/resources/south-clinic/squads/front-desk.yml new file mode 100644 index 0000000..46525b7 --- /dev/null +++ b/examples/blocks/resources/south-clinic/squads/front-desk.yml @@ -0,0 +1,8 @@ +name: Front Desk +members: + - assistantId: triage + - assistantId: pharmacy +membersOverrides: + $include: call-settings + variableValues: + practice_name: South Clinic diff --git a/examples/blocks/resources/south-clinic/templates/blocks/red-flags.md b/examples/blocks/resources/south-clinic/templates/blocks/red-flags.md new file mode 100644 index 0000000..e8ecaf7 --- /dev/null +++ b/examples/blocks/resources/south-clinic/templates/blocks/red-flags.md @@ -0,0 +1,4 @@ +@include red-flags + +{{practice_name}} has a walk-in centre next door, open until 8pm. If the +caller is not sure it is an emergency, tell them they can go there. diff --git a/examples/blocks/templates/blocks/red-flags.md b/examples/blocks/templates/blocks/red-flags.md new file mode 100644 index 0000000..28343b6 --- /dev/null +++ b/examples/blocks/templates/blocks/red-flags.md @@ -0,0 +1,5 @@ +# Emergencies + +If the caller mentions chest pain, difficulty breathing or signs of a stroke, +stop what you are doing and tell them to hang up and call 999 now. Then offer +to connect them to {{practice_name}} reception. diff --git a/examples/blocks/templates/config/call-settings.yml b/examples/blocks/templates/config/call-settings.yml new file mode 100644 index 0000000..a5a4425 --- /dev/null +++ b/examples/blocks/templates/config/call-settings.yml @@ -0,0 +1,4 @@ +silenceTimeoutSeconds: 20 +maxDurationSeconds: 900 +voice: + $include: voice diff --git a/examples/blocks/templates/config/voice.yml b/examples/blocks/templates/config/voice.yml new file mode 100644 index 0000000..518f9ec --- /dev/null +++ b/examples/blocks/templates/config/voice.yml @@ -0,0 +1,2 @@ +provider: 11labs +voiceId: sarah diff --git a/improvements.md b/improvements.md index ef4ca0e..b8f9161 100644 --- a/improvements.md +++ b/improvements.md @@ -2062,6 +2062,94 @@ dependency check, as `check-payload.ts` already does. --- +## 39. Shared prompt text and config had no home, and pull would erase any include marker + +### Problem + +Assistants in one org, and the same assistant across orgs, repeat the same +paragraphs (safety wording, tone) and the same settings (voice, transcriber). +There was no way to keep one copy. YAML anchors only reach inside one file and +are expanded by the next pull; merge keys (`<<`) are not processed at all and +are sent to the API as a field; Liquid `{% include %}` renders as nothing on a +call. + +### Current behavior (Verified) + +Before blocks: a marker of any kind in a resource file was sent to Vapi as +literal text. Had the loader expanded it, `pull` would still have destroyed +it: pull rewrites a file from the platform copy not only when the dashboard +is ahead but whenever the two sides agree (the `clean` case falls through to +the write), and under `--force` and `--resolve=theirs`. Promotion compares +file text, so a change to a shared file would never have reached the next org. + +### Risk + +Safety wording drifts between copies. With a naive include, the first pull +after a push silently turns every marker back into a copy. + +### Current mitigation + +None needed; fixed. + +### Possible fix (landed) + +Blocks (`src/blocks.ts`, `docs/guides/blocks.md`): `@include ` in system +prompts and `$include` in mappings, looked up in `resources//templates/` +then `templates/`, expanded in the loader. Pull never writes a file that uses +blocks and saves the dashboard version beside it instead. Push records the +block content each resource went up with in +`resources//.vapi-blocks.json`; `npm run sync-blocks` and the promotion +plan (`sync` entries) read it to find resources a block edit has not reached. + +### Status + +**[RESOLVED 2026-11-16] (#94)** + +--- + +## 40. An org with no credentials is re-bootstrapped on every push, which disables the drift check + +### Problem + +The push drift check refuses to overwrite a resource that was edited in the +dashboard since the last pull. In an org that has no credentials stored in +Vapi, it never refuses. + +### Current behavior (Verified) + +`maybeBootstrapState` in `src/push.ts` treats a state file with an empty +`credentials` section as uninitialized and runs a bootstrap pull before +applying. A bootstrap pull writes every resource's baseline as the hash of +the platform's current content. The drift check then compares the platform +with a baseline taken a moment earlier, finds them equal, and pushes. Seen +in `tests/blocks-sync.test.ts`: with the stub's `/credential` route removed, +the dashboard-edited assistant is PATCHed with no drift message; with one +credential present it is refused. + +### Risk + +A dashboard edit in such an org is overwritten silently by `push`, `apply` +and `sync-blocks --apply`, with no prompt in a terminal and no block in CI. +An org that uses only Vapi-provided models, voices and numbers can have no +credentials. + +### Current mitigation + +None. `npm run pull -- ` first still shows a dashboard edit before it is +overwritten, for anyone who reads the output. + +### Possible fix + +Decide "uninitialized" from the resource sections alone (and from a marker +that a pull has completed), not from the credentials section being empty; or +have a bootstrap pull leave existing baselines alone. + +### Status + +**Open.** + +--- + ## Out of scope (intentionally not improvements) - **State file is identity-only and not git-ignored.** It's intentionally diff --git a/package.json b/package.json index 50ad256..bc8336d 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,7 @@ "check": "tsx src/check-cmd.ts", "rollback": "tsx src/rollback-cmd.ts", "promote": "tsx src/promote-cmd.ts", + "sync-blocks": "tsx src/sync-blocks-cmd.ts", "build": "tsc --noEmit", "test": "node --import tsx --import ./tests/no-vapi-api.ts --test tests/*.test.ts" }, diff --git a/src/blocks.ts b/src/blocks.ts new file mode 100644 index 0000000..8289380 --- /dev/null +++ b/src/blocks.ts @@ -0,0 +1,350 @@ +// Blocks: text and config shared between resource files. +// +// A prompt block is a Markdown file under templates/blocks/. A system prompt +// pulls it in with a line that holds only "@include ". A config block +// is a YAML file under templates/config/. Any mapping in a resource pulls it +// in with a "$include" key, and the mapping's own keys win over the block's. +// +// Both kinds are looked up in resources//templates/ first, then in +// templates/ at the repository root, so an org can replace a shared block. +// A block inside the org folder that includes its own name gets the shared +// one, which lets it extend the shared block rather than copy it. +// +// Expansion happens when a resource file is loaded, so validate, push, PR +// checks, drift hashes and the audit all see the expanded resource. Nothing +// inside a block is interpreted: Liquid variables reach Vapi as written. +// The marker is deliberately not Liquid, because Vapi renders "include" and +// "render" tags as nothing on a live call. +// +// Config-free, like resource-parse.ts, so every org can be read in one run. + +import { createHash } from "node:crypto"; +import { existsSync, readFileSync, writeFileSync } from "node:fs"; +import { dirname, join, relative, resolve } from "node:path"; +import { isDeepStrictEqual } from "node:util"; +import { parse as parseYaml } from "yaml"; + +export const TEMPLATES_FOLDER = "templates"; +export const INCLUDE_KEY = "$include"; + +const INCLUDE_LINE = /^[ \t]*@include[ \t]+(\S+)[ \t]*\r?$/; +const INCLUDE_LINE_ANYWHERE = /^[ \t]*@include[ \t]+\S+[ \t]*\r?$/m; +const BLOCK_NAME = /^[A-Za-z0-9][A-Za-z0-9_-]*(\/[A-Za-z0-9][A-Za-z0-9_-]*)*$/; + +interface BlockKind { + label: string; + folder: string; + extensions: string[]; +} + +const PROMPT: BlockKind = { + label: "prompt block", + folder: "blocks", + extensions: [".md"], +}; +const CONFIG: BlockKind = { + label: "config block", + folder: "config", + extensions: [".yml", ".yaml"], +}; + +// The two folders a block is looked up in, in order. +export interface BlockLayers { + org: string; + shared: string; +} + +// /resources/ → /resources//templates and /templates. +export function blockLayers(resourcesDir: string): BlockLayers { + return { + org: join(resourcesDir, TEMPLATES_FOLDER), + shared: resolve(resourcesDir, "..", "..", TEMPLATES_FOLDER), + }; +} + +// Block file → hash of its content, for every block a resource resolved to, +// nested ones included. Keys are paths inside a templates folder +// ("blocks/red-flags.md"), the same whichever layer supplied the file. +export type BlockUsage = Record; + +interface BlockFile { + key: string; + path: string; + content: string; +} + +interface Expansion { + layers: BlockLayers; + usage: Map; + // Block files being expanded right now, outermost first. + stack: BlockFile[]; +} + +type Mapping = Record; + +function isMapping(value: unknown): value is Mapping { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function isSystemMessage( + value: Mapping, +): value is Mapping & { content: string } { + return value.role === "system" && typeof value.content === "string"; +} + +function blockFind( + kind: BlockKind, + name: string, + expansion: Expansion, +): BlockFile { + if (!BLOCK_NAME.test(name)) + throw new Error( + `Invalid ${kind.label} name "${name}": use the file name without its extension, for example "red-flags" or "safety/red-flags"`, + ); + const { layers, stack } = expansion; + const from = stack[stack.length - 1]?.path; + const root = dirname(layers.shared); + const searched: string[] = []; + for (const layer of [layers.org, layers.shared]) { + for (const extension of kind.extensions) { + const path = join(layer, kind.folder, `${name}${extension}`); + // An org block that includes its own name extends the shared one. + if (layer === layers.org && path === from) continue; + searched.push(relative(root, path)); + if (!existsSync(path)) continue; + const block = { + key: `${kind.folder}/${name}${extension}`, + path, + content: readFileSync(path, "utf8"), + }; + if (stack.some((open) => open.path === path)) + throw new Error( + `Blocks include each other in a loop: ${[...stack, block].map((open) => relative(root, open.path)).join(" → ")}`, + ); + // An org block that extends the shared one uses both files. + const hash = createHash("sha256").update(block.content); + const outer = expansion.usage.get(block.key); + if (outer !== undefined) hash.update(outer); + expansion.usage.set(block.key, hash.digest("hex").slice(0, 16)); + return block; + } + } + throw new Error( + `Unknown ${kind.label} "${name}". Looked for:\n${searched.map((path) => ` - ${path}`).join("\n")}`, + ); +} + +function blockExpand( + block: BlockFile, + expansion: Expansion, + expand: () => T, +): T { + expansion.stack.push(block); + try { + return expand(); + } finally { + expansion.stack.pop(); + } +} + +function textExpand(text: string, expansion: Expansion): string { + if (!INCLUDE_LINE_ANYWHERE.test(text)) return text; + return text + .split("\n") + .map((line) => { + const name = line.match(INCLUDE_LINE)?.[1]; + if (name === undefined) return line; + const block = blockFind(PROMPT, name, expansion); + // The block's own text only: no byte-order mark, no blank lines around it. + const body = block.content + .replace(/^\uFEFF/, "") + .replace(/^(?:[ \t]*\r?\n)+/, "") + .trimEnd(); + return blockExpand(block, expansion, () => textExpand(body, expansion)); + }) + .join("\n"); +} + +// Two blocks included side by side: mappings combine, and anything else set +// by both must be identical, so the result never depends on their order. +function blocksCombine( + left: Mapping, + right: Mapping, + names: string, + path: string[], +): Mapping { + const combined: Mapping = { ...left }; + for (const [key, value] of Object.entries(right)) { + const existing = combined[key]; + if (!(key in combined)) combined[key] = value; + else if (isMapping(existing) && isMapping(value)) + combined[key] = blocksCombine(existing, value, names, [...path, key]); + else if (!isDeepStrictEqual(existing, value)) + throw new Error( + `Config blocks ${names} both set "${[...path, key].join(".")}" to different values. Set it in one block, or in the file that includes them.`, + ); + } + return combined; +} + +// The including mapping over its blocks: mappings merge key by key, and any +// other value the file sets (a list included) replaces the block's whole. +function fileOverBlock(block: Mapping, file: Mapping): Mapping { + const merged: Mapping = { ...block }; + for (const [key, value] of Object.entries(file)) { + const inherited = merged[key]; + merged[key] = + isMapping(inherited) && isMapping(value) + ? fileOverBlock(inherited, value) + : value; + } + return merged; +} + +function includeNames(value: unknown): string[] { + const names: unknown[] = Array.isArray(value) ? value : [value]; + if ( + names.length === 0 || + names.some((name) => typeof name !== "string" || name === "") + ) + throw new Error( + `"${INCLUDE_KEY}" takes a config block name or a list of names, got ${JSON.stringify(value)}`, + ); + if (new Set(names).size !== names.length) + throw new Error(`"${INCLUDE_KEY}" lists the same config block twice`); + return names as string[]; +} + +function configExpand(value: unknown, expansion: Expansion): unknown { + if (Array.isArray(value)) + return value.map((item) => configExpand(item, expansion)); + if (!isMapping(value)) return value; + const own: Mapping = {}; + for (const [key, child] of Object.entries(value)) + if (key !== INCLUDE_KEY) own[key] = configExpand(child, expansion); + if (!(INCLUDE_KEY in value)) return own; + if ("<<" in value) + throw new Error( + `A mapping uses both "${INCLUDE_KEY}" and a YAML merge key ("<<"). The engine does not process merge keys; keep "${INCLUDE_KEY}" only.`, + ); + const names = includeNames(value[INCLUDE_KEY]); + const included = names.map((name) => { + const block = blockFind(CONFIG, name, expansion); + const parsed: unknown = parseYaml(block.content); + if (parsed === null || parsed === undefined) + throw new Error(`Config block "${name}" is empty`); + return blockExpand(block, expansion, () => configExpand(parsed, expansion)); + }); + // On its own, "$include" stands for the block's value, whatever its shape. + if (Object.keys(own).length === 0 && included.length === 1) + return included[0]; + const quoted = names.map((name) => `"${name}"`).join(" and "); + let combined: Mapping = {}; + for (const [index, block] of included.entries()) { + if (!isMapping(block)) + throw new Error( + `Config block "${names[index]}" is not a mapping, so it can only be included on its own: no other keys beside "${INCLUDE_KEY}", and no other block in the list.`, + ); + combined = blocksCombine(combined, block, quoted, []); + } + return fileOverBlock(combined, own); +} + +function promptsExpand(value: unknown, expansion: Expansion): unknown { + if (Array.isArray(value)) + return value.map((item) => promptsExpand(item, expansion)); + if (!isMapping(value)) return value; + const output: Mapping = {}; + for (const [key, child] of Object.entries(value)) + output[key] = promptsExpand(child, expansion); + if (isSystemMessage(output)) + output.content = textExpand(output.content, expansion); + return output; +} + +// Whether a parsed resource file uses a block anywhere. Pull asks this of the +// file as written, before it decides whether it may rewrite that file. +export function blocksUsedIn(value: unknown): boolean { + if (Array.isArray(value)) return value.some(blocksUsedIn); + if (!isMapping(value)) return false; + if (INCLUDE_KEY in value) return true; + if (isSystemMessage(value) && INCLUDE_LINE_ANYWHERE.test(value.content)) + return true; + return Object.values(value).some(blocksUsedIn); +} + +// Expand every block a parsed resource uses. A resource that uses none is +// returned as it came in. +export function blocksExpand( + data: T, + layers: BlockLayers, +): { data: T; usage: BlockUsage } { + if (!blocksUsedIn(data)) return { data, usage: {} }; + const expansion: Expansion = { layers, usage: new Map(), stack: [] }; + const expanded = promptsExpand(configExpand(data, expansion), expansion); + const usage: BlockUsage = {}; + for (const key of [...expansion.usage.keys()].sort()) + usage[key] = expansion.usage.get(key)!; + return { data: expanded as T, usage }; +} + +// ───────────────────────────────────────────────────────────────────────────── +// Sync record +// +// resources//.vapi-blocks.json records, for each resource that uses +// blocks, the block content it was last pushed with. Push writes it and it is +// committed. A resource whose blocks no longer match its entry still runs the +// old block text on the platform: that is what "npm run sync-blocks" lists +// and what promotion adds to its plan. The file text of such a resource has +// not changed, so nothing else in git shows that it needs a push. +// ───────────────────────────────────────────────────────────────────────────── + +export const BLOCKS_RECORD_FILE = ".vapi-blocks.json"; + +// "/" → the blocks it was pushed with. +export type BlocksRecord = Record; + +export function blocksRecordRead(resourcesDir: string): BlocksRecord { + const path = join(resourcesDir, BLOCKS_RECORD_FILE); + if (!existsSync(path)) return {}; + const parsed: unknown = JSON.parse(readFileSync(path, "utf8")); + if (!isMapping(parsed)) throw new Error(`${path} must hold a JSON object`); + return parsed as BlocksRecord; +} + +// The block files that differ between what a resource uses now and what it +// was last pushed with; empty when it is in sync. +export function blocksChanged( + usage: BlockUsage, + recorded: BlockUsage | undefined, +): string[] { + const keys = new Set([...Object.keys(usage), ...Object.keys(recorded ?? {})]); + return [...keys].filter((key) => usage[key] !== recorded?.[key]).sort(); +} + +// Record what the pushed resources were pushed with, and drop entries for +// resources that no longer exist or no longer use blocks. Entries for +// resources this push did not touch are kept: they may still be out of date. +export function blocksRecordUpdate( + resourcesDir: string, + current: Map, + pushed: Iterable, +): void { + const before = blocksRecordRead(resourcesDir); + const after: BlocksRecord = { ...before }; + for (const key of pushed) { + const usage = current.get(key); + if (usage) after[key] = usage; + } + for (const key of Object.keys(after)) { + const usage = current.get(key); + if (!usage || Object.keys(usage).length === 0) delete after[key]; + } + if (isDeepStrictEqual(before, after)) return; + const sorted: BlocksRecord = {}; + for (const key of Object.keys(after).sort()) sorted[key] = after[key]!; + writeFileSync( + join(resourcesDir, BLOCKS_RECORD_FILE), + `${JSON.stringify(sorted, null, 2)}\n`, + ); +} diff --git a/src/check-select.ts b/src/check-select.ts index f3d91da..240aa8f 100644 --- a/src/check-select.ts +++ b/src/check-select.ts @@ -12,6 +12,9 @@ const ENGINE_PATTERNS = [ CHECKS_CONFIG_FILE, "promotion.yml", "src/**", + // Shared blocks are expanded into any org's resources. An org's own blocks + // sit under resources//, which each check already watches. + "templates/**", "package.json", "package-lock.json", ]; diff --git a/src/promote-cmd.ts b/src/promote-cmd.ts index a219b17..bdf3c43 100644 --- a/src/promote-cmd.ts +++ b/src/promote-cmd.ts @@ -265,7 +265,9 @@ async function transitionRun( `\n${transition.pipeline}: ${transition.source} → ${transition.target}`, ); for (const change of plan.changes) - console.log(` ${change.kind.padEnd(6)} ${change.path}`); + console.log( + ` ${change.kind.padEnd(6)} ${change.path}${change.blocks ? ` (block changed: ${change.blocks.join(", ")})` : ""}`, + ); if (plan.changes.length === 0) console.log(" no changes"); if (plan.changes.length === 0) return false; const check = gates.checks.get(transition.source); diff --git a/src/promotion.ts b/src/promotion.ts index eb13bae..fc948ae 100644 --- a/src/promotion.ts +++ b/src/promotion.ts @@ -11,7 +11,18 @@ import { dirname, extname, join, relative } from "node:path"; import { pathToFileURL } from "node:url"; import { isDeepStrictEqual } from "node:util"; import { parse as parseYaml, stringify as stringifyYaml } from "yaml"; -import { FOLDER_MAP, VALID_EXTENSIONS } from "./resource-parse.ts"; +import { + type BlockUsage, + blockLayers, + blocksChanged, + blocksExpand, + blocksRecordRead, +} from "./blocks.ts"; +import { + FOLDER_MAP, + markdownResourceParse, + VALID_EXTENSIONS, +} from "./resource-parse.ts"; import type { ResourceType, StateFile } from "./types.ts"; type BindingPolicy = "bind" | "omit"; @@ -46,9 +57,13 @@ export interface PromotionConfig { } export interface PromotionChange { - kind: "create" | "update" | "delete"; + // "sync": the target's file is already identical, but a block it uses has + // changed since the target was last pushed, so it needs pushing again. + kind: "create" | "update" | "delete" | "sync"; path: string; content?: string; + // For "sync": the block files that changed. + blocks?: string[]; } export interface PromotionPlan { @@ -597,6 +612,9 @@ export async function promotionPlanBuild( options.sourceState, ); const targetByPath = new Map(target.map((file) => [file.path, file])); + const targetDir = join(options.rootDir, "resources", options.target); + const targetLayers = blockLayers(targetDir); + const targetRecord = blocksRecordRead(targetDir); const sourcePaths = new Set([...wanted.values()].map((file) => file.path)); const changes: PromotionChange[] = []; for (const file of wanted.values()) { @@ -621,12 +639,38 @@ export async function promotionPlanBuild( ? file.content : renderContent(transformed, file.path, parsed.body); const existing = targetByPath.get(file.path); - if (!existing || existing.content !== content) + if (!existing || existing.content !== content) { changes.push({ kind: existing ? "update" : "create", path: file.path, content, }); + continue; + } + // Same file text in both orgs. The target still needs a push when a + // block the file uses has changed since the target last received it: + // blocks are expanded with the target's own folder first, at push time. + let usage: BlockUsage; + try { + usage = blocksExpand( + extname(file.path) === ".ts" + ? parsed.data + : extname(file.path) === ".md" + ? markdownResourceParse(content) + : parseYaml(content), + targetLayers, + ).usage; + } catch (error) { + throw new Error( + `${file.path} can't be expanded for ${options.target}: ${error instanceof Error ? error.message : error}`, + ); + } + const blocks = blocksChanged( + usage, + targetRecord[`${FOLDER_MAP[file.type]}/${file.id}`], + ); + if (Object.keys(usage).length > 0 && blocks.length > 0) + changes.push({ kind: "sync", path: file.path, blocks }); } for (const file of target) if ( @@ -647,6 +691,7 @@ export async function promotionPlanApply(plan: PromotionPlan): Promise { await rm(path, { force: true }); continue; } + if (change.kind === "sync") continue; await mkdir(dirname(path), { recursive: true }); await writeFile(path, change.content ?? ""); } diff --git a/src/pull.ts b/src/pull.ts index 62b487f..14f44da 100644 --- a/src/pull.ts +++ b/src/pull.ts @@ -35,6 +35,7 @@ import { } from "./drift.ts"; import { readBaseline, writeBaseline } from "./hash-store.ts"; import { assertStateMigrated } from "./migrate-hash-store.ts"; +import { resourceFileUsesBlocks } from "./resource-parse.ts"; import { formatRecanonicalizeReport, recanonicalizeStateKeys, @@ -595,6 +596,110 @@ export async function writeDashboardBackup( ); } +// A local file that uses blocks is never rewritten by pull. The platform +// holds the expanded resource, so writing it back would replace every marker +// with a copy of the block's text and cut the file off from later block +// edits. When the dashboard version would have been written, it goes beside +// the file as a backup copy and the person merges the edit by hand. +function localBlocksFile( + resourceType: ResourceType, + resourceId: string, +): string | undefined { + const path = findLocalResourcePath(FOLDER_MAP[resourceType], resourceId); + return path && resourceFileUsesBlocks(path) ? path : undefined; +} + +async function blocksFileKeep( + resourceType: ResourceType, + resourceId: string, + resource: VapiResource, + state: StateFile, + localFile: string, +): Promise { + const copy = await writeDashboardBackup( + resourceType, + resourceId, + resource, + state, + ); + console.warn( + ` ⚠️ ${resourceId}: the dashboard version differs, but the file uses blocks, so it was not overwritten.`, + ); + console.warn( + ` Dashboard version saved to ${relative(BASE_DIR, copy)}.`, + ); + console.warn( + ` Move the dashboard edit into the file or into the block it belongs to, then run: npm run apply -- ${VAPI_ENV} --resolve=ours ${relative(BASE_DIR, localFile)}`, + ); +} + +// Pull one resource whose local file uses blocks. Local and platform are +// compared in expanded form, like any other resource, but no outcome writes +// the file. The drift baseline moves only when the two sides already agree, +// so an unresolved dashboard edit is still drift at the next pull and push. +async function blocksFilePull(options: { + resourceType: ResourceType; + resourceId: string; + resource: VapiResource; + state: StateFile; + credReverse: Map; + localFile: string; + force?: boolean; + driftCounts?: DriftDirectionCounts; + bothDiverged?: BothDivergedResource[]; +}): Promise { + const { resourceType, resourceId, resource, state, localFile } = options; + const localHash = hashLocalResource(resourceType, resourceId); + if (localHash === null) { + console.warn( + ` ⚠️ ${resourceId}: uses blocks that could not be expanded, file left as written (run npm run validate -- ${VAPI_ENV})`, + ); + return; + } + const platformHash = hashPayload( + canonicalizeForHash(resource, state, options.credReverse), + ); + const baseline = readBaseline(VAPI_ENV, resource.id); + const direction = classifyDrift({ + localHash, + lastPulledHash: baseline, + platformHash, + }); + if (!options.force && options.driftCounts) options.driftCounts[direction]++; + + if (localHash === platformHash) { + console.log( + ` 🧩 ${resourceId} (uses blocks, matches the dashboard — file left as written)`, + ); + await writeBaseline(VAPI_ENV, resource.id, platformHash); + return; + } + if (options.force || direction === "dashboard-ahead") { + await blocksFileKeep(resourceType, resourceId, resource, state, localFile); + return; + } + if (direction === "both-diverged") { + options.bothDiverged?.push({ + resourceType, + resourceId, + resource, + localHash, + platformHash, + lastPulledHash: baseline!, + }); + return; + } + if (direction === "local-ahead") { + console.log( + ` ⬆️ ${resourceId} (local ahead of dashboard) ${formatDriftLabel(direction)}`, + ); + return; + } + console.log( + ` ✏️ ${resourceId} (uses blocks and differs from the dashboard, no drift baseline yet — local file kept)`, + ); +} + // ───────────────────────────────────────────────────────────────────────────── // Pull Functions // ───────────────────────────────────────────────────────────────────────────── @@ -886,6 +991,26 @@ export async function pullResourceType( } } + const blocksFile = bootstrap + ? undefined + : localBlocksFile(resourceType, resourceId); + if (blocksFile) { + await blocksFilePull({ + resourceType, + resourceId, + resource, + state, + credReverse, + localFile: blocksFile, + force, + driftCounts, + bothDiverged, + }); + upsertState(newStateSection, resourceId, { uuid: resource.id }); + skipped++; + continue; + } + let skipLegacyPreserveChecks = false; // Hash-based drift direction when a baseline exists in the hash store. @@ -1214,6 +1339,22 @@ async function resolveBothDivergedResources(options: { continue; } + // "theirs" would write the dashboard version over the local file. For a + // file that uses blocks that is refused: the conflict stays open, with + // the dashboard version beside the file. + const blocksFile = localBlocksFile(entry.resourceType, entry.resourceId); + if (blocksFile) { + await blocksFileKeep( + entry.resourceType, + entry.resourceId, + entry.resource, + state, + blocksFile, + ); + upsertState(section, entry.resourceId, { uuid: entry.resource.id }); + continue; + } + const withCredNames = canonicalizeForHash( entry.resource, state, diff --git a/src/push.ts b/src/push.ts index c22f41d..5dce5c5 100644 --- a/src/push.ts +++ b/src/push.ts @@ -2,6 +2,7 @@ import { select } from "@inquirer/prompts"; import { relative, resolve } from "path"; import { fileURLToPath } from "url"; import { getDryRunCounts, VapiApiError, vapiRequest } from "./api.ts"; +import { type BlockUsage, blocksRecordUpdate } from "./blocks.ts"; import { canonicalizeForHash, type VapiResource } from "./canonical.ts"; import { ALLOW_NEW_FILES, @@ -212,6 +213,15 @@ async function writeAssistantVersionMetadata(options: { } } +// Every resource this run created or updated on the platform, as +// "/". The blocks record marks a resource as in sync with +// its blocks only when it is in here: a skipped or refused push is not. +const pushedResources = new Set(); + +function pushedRecord(type: ResourceType | undefined, resourceId: string) { + if (type) pushedResources.add(`${FOLDER_MAP[type]}/${resourceId}`); +} + async function upsertResourceWithStateRecovery(options: { resourceLabel: string; resourceId: string; @@ -251,6 +261,7 @@ async function upsertResourceWithStateRecovery(options: { after: result, }); } + pushedRecord(RESOURCE_LABEL_TO_TYPE[resourceLabel], resourceId); return result.id; } @@ -405,6 +416,7 @@ async function upsertResourceWithStateRecovery(options: { after: result, }); } + pushedRecord(RESOURCE_LABEL_TO_TYPE[resourceLabel], resourceId); return existingUuid; } catch (error) { if (!(error instanceof VapiApiError) || error.statusCode !== 404) { @@ -1022,11 +1034,13 @@ export async function applyEval( updatePayload, ); await writeBaselineFromResponse(existingUuid, result, state); + pushedRecord("evals", resourceId); return existingUuid; } else { console.log(` ✨ Creating eval: ${resourceId}`); const result = await vapiRequest("POST", "/eval", payload); await writeBaselineFromResponse(result.id, result, state); + pushedRecord("evals", resourceId); return result.id; } } @@ -1619,6 +1633,11 @@ async function main(): Promise { // merge into existing on-disk state instead of rewriting wholesale. const touched: TouchedSets = emptyTouchedSets(); + // The blocks every local resource uses, keyed "/". Set + // once the files have loaded, so a run that fails before then leaves the + // blocks record alone. + let blocksUsed: Map | undefined; + // Track what was applied for summary const applied: Record = { tools: 0, @@ -1704,6 +1723,14 @@ async function main(): Promise { simulationSuites: allSimulationSuitesRaw, evals: allEvalsRaw, }; + blocksUsed = new Map( + (Object.keys(loadedResources) as ResourceType[]).flatMap((type) => + loadedResources[type].map((file) => [ + `${FOLDER_MAP[type]}/${file.resourceId}`, + file.blocks ?? {}, + ]), + ), + ); state = await maybeBootstrapState(loadedResources, state); @@ -2294,6 +2321,18 @@ async function main(): Promise { ` Local state may be out of sync with platform. Run \`npm run pull -- ${VAPI_ENV} --bootstrap\` to recover.`, ); } + // Record the block content each pushed resource went up with, so + // sync-blocks and promotion can tell which resources a later block + // edit has not reached yet. + try { + if (blocksUsed) + blocksRecordUpdate(RESOURCES_DIR, blocksUsed, pushedResources); + } catch (recordError) { + console.error( + "\n⚠️ Failed to update the blocks record after apply:", + recordError instanceof Error ? recordError.message : recordError, + ); + } } } } diff --git a/src/resource-parse.ts b/src/resource-parse.ts index 0af7325..d210a5d 100644 --- a/src/resource-parse.ts +++ b/src/resource-parse.ts @@ -10,6 +10,12 @@ import { existsSync, readFileSync } from "fs"; import { readdir, readFile, stat } from "fs/promises"; import { extname, join, relative } from "path"; import { parse as parseYaml } from "yaml"; +import { + type BlockUsage, + blockLayers, + blocksExpand, + blocksUsedIn, +} from "./blocks.ts"; import { isBackupCopyFile } from "./slug-utils.ts"; import type { ResourceFile, ResourceType } from "./types.ts"; @@ -104,9 +110,8 @@ export function markdownResourceParse( return config; } -export function parseResourceDataFromFile( - filePath: string, -): Record { +// A `.md`, `.yml` or `.yaml` resource file as written, blocks not expanded. +function resourceFileParse(filePath: string): Record { const ext = extname(filePath); if (ext === ".md") { @@ -124,6 +129,36 @@ export function parseResourceDataFromFile( return data; } +// The resource a file describes, blocks expanded: what push sends, and so +// what every drift hash is computed from. Blocks are looked up from +// `resourcesDir` (the org's folder), so a file that uses them needs it. +export function parseResourceDataFromFile( + filePath: string, + resourcesDir?: string, +): Record { + const data = resourceFileParse(filePath); + if (resourcesDir) return blocksExpand(data, blockLayers(resourcesDir)).data; + if (blocksUsedIn(data)) + throw new Error( + `${filePath} uses blocks; pass its org folder to expand them`, + ); + return data; +} + +// Whether a resource file uses blocks. Pull must never rewrite such a file: +// the platform holds the expanded text, so writing it back would replace +// every marker with a copy of the block. A file that can't be parsed counts +// as using blocks when its text looks like it does. +export function resourceFileUsesBlocks(filePath: string): boolean { + try { + return blocksUsedIn(resourceFileParse(filePath)); + } catch { + return /^\s*(?:@include\s+\S|(?:- )?["']?\$include["']?\s*:)/m.test( + readFileSync(filePath, "utf-8"), + ); + } +} + /** * Recursively scan a directory for resource files (.yml, .yaml, .ts) * Warns about unsupported files found in resource directories @@ -196,6 +231,7 @@ export async function resourceDirLoad( } const filePaths = await scanDirectory(resourceDir, resourceDir); + const layers = blockLayers(resourcesDir); const resources: ResourceFile[] = []; const seenIds = new Map(); // resourceId -> filePath @@ -274,7 +310,16 @@ export async function resourceDirLoad( } } - resources.push({ resourceId, filePath, data }); + let blocks: BlockUsage; + try { + ({ data, usage: blocks } = blocksExpand(data, layers)); + } catch (error) { + throw new Error( + `Failed to expand blocks in "${folderPath}/${relativePath}": ${error instanceof Error ? error.message : error}`, + ); + } + + resources.push({ resourceId, filePath, data, blocks }); if (!options.quiet) console.log(` 📦 Loaded ${resourceId}`); } @@ -286,6 +331,8 @@ export interface OrgResource { id: string; filePath: string; data: Record; + // The blocks the file resolved to; empty when it uses none. + blocks: BlockUsage; } export interface OrgResourcesReadOptions { @@ -319,6 +366,7 @@ export async function orgResourcesRead( id: file.resourceId, filePath: file.filePath, data: file.data, + blocks: file.blocks ?? {}, }); } } diff --git a/src/resources.ts b/src/resources.ts index 6ac5213..2e21b4f 100644 --- a/src/resources.ts +++ b/src/resources.ts @@ -2,6 +2,7 @@ import { existsSync } from "fs"; import { readFile } from "fs/promises"; import { basename, extname, join, relative, resolve } from "path"; import { parse as parseYaml } from "yaml"; +import { blockLayers, blocksExpand } from "./blocks.ts"; import { BASE_DIR, matchesIgnore, RESOURCES_DIR } from "./config.ts"; import { FOLDER_MAP, @@ -42,7 +43,7 @@ export function hashLocalResource( const filePath = findLocalResourceFile(type, resourceId); if (!filePath) return null; try { - return hashPayload(parseResourceDataFromFile(filePath)); + return hashPayload(parseResourceDataFromFile(filePath, RESOURCES_DIR)); } catch { return null; } @@ -283,10 +284,19 @@ export async function loadSingleResource( } } + let blocks: Record; + try { + ({ data, usage: blocks } = blocksExpand(data, blockLayers(RESOURCES_DIR))); + } catch (error) { + throw new Error( + `Failed to expand blocks in "${filePath}": ${error instanceof Error ? error.message : error}`, + ); + } + console.log(` 📦 Loaded ${resourceId} (${resourceType})`); return { type: resourceType, - resource: { resourceId, filePath: absolutePath, data }, + resource: { resourceId, filePath: absolutePath, data, blocks }, }; } diff --git a/src/sync-blocks-cmd.ts b/src/sync-blocks-cmd.ts new file mode 100644 index 0000000..015fc53 --- /dev/null +++ b/src/sync-blocks-cmd.ts @@ -0,0 +1,182 @@ +// CLI entry: `npm run sync-blocks [-- --org ] [--apply]` +// +// A block edit changes every resource that includes the block, in every org, +// without changing any of their files. This command finds those resources: +// the ones whose blocks differ from what the blocks record +// (resources//.vapi-blocks.json) says they were last pushed with. +// +// Without --apply it only lists them, reading files and nothing else. With +// --apply it runs `apply` for each org, scoped to those files, so the usual +// validate → pull → push path decides what reaches the platform: a resource +// that was edited in the dashboard is refused by the drift check there, and +// stays listed here. + +import { spawnSync } from "node:child_process"; +import { existsSync, readdirSync, statSync } from "node:fs"; +import { join, relative, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { blocksChanged, blocksRecordRead } from "./blocks.ts"; +import { FOLDER_MAP, orgResourcesRead } from "./resource-parse.ts"; + +export interface BlocksStaleResource { + org: string; + // Path from the repository root. + path: string; + // The block files that changed since the resource was last pushed. + blocks: string[]; +} + +export interface SyncBlocksDeps { + rootDir: string; + // Deploy the given files of one org; returns whether the run succeeded. + applyRun: (org: string, paths: string[]) => boolean; +} + +const USAGE = "Usage: npm run sync-blocks [-- --org ] [--apply]"; + +function orgsRead(rootDir: string): string[] { + const resources = join(rootDir, "resources"); + if (!existsSync(resources)) return []; + return readdirSync(resources) + .filter( + (entry) => + !entry.startsWith(".") && statSync(join(resources, entry)).isDirectory(), + ) + .sort(); +} + +// Every resource of one org whose blocks changed since it was last pushed. +export async function blocksStaleRead( + rootDir: string, + org: string, +): Promise { + const resourcesDir = join(rootDir, "resources", org); + const record = blocksRecordRead(resourcesDir); + const stale: BlocksStaleResource[] = []; + for (const resource of (await orgResourcesRead(rootDir, org)).values()) { + if (Object.keys(resource.blocks).length === 0) continue; + const blocks = blocksChanged( + resource.blocks, + record[`${FOLDER_MAP[resource.type]}/${resource.id}`], + ); + if (blocks.length > 0) + stale.push({ org, path: relative(rootDir, resource.filePath), blocks }); + } + return stale.sort((left, right) => left.path.localeCompare(right.path)); +} + +function argumentsParse(args: string[]): { orgs: string[]; apply: boolean } { + const parsed = { orgs: [] as string[], apply: false }; + for (let index = 0; index < args.length; index++) { + const argument = args[index]; + if (argument === "--apply") parsed.apply = true; + else if (argument === "--org" && args[index + 1]) + parsed.orgs.push(args[++index]!); + else + throw new Error( + `Unrecognized or incomplete argument: ${argument}\n${USAGE}`, + ); + } + return parsed; +} + +function applyRun(rootDir: string, org: string, paths: string[]): boolean { + const result = spawnSync( + process.execPath, + ["--import", "tsx", "src/apply.ts", org, ...paths], + { cwd: rootDir, stdio: "inherit" }, + ); + return result.status === 0; +} + +// Returns the process exit code: 1 when any resource that needs a sync was +// not synced, or an org's files could not be read. +export async function syncBlocksRun( + args = process.argv.slice(2), + overrides: Partial = {}, +): Promise { + const rootDir = + overrides.rootDir ?? + resolve( + process.env.VAPI_GITOPS_ROOT ?? + fileURLToPath(new URL("..", import.meta.url)), + ); + const deploy = + overrides.applyRun ?? + ((org: string, paths: string[]) => applyRun(rootDir, org, paths)); + const parsed = argumentsParse(args); + const known = orgsRead(rootDir); + const unknown = parsed.orgs.filter((org) => !known.includes(org)); + if (unknown.length > 0) + throw new Error(`No resources/${unknown[0]}/ folder in this repository`); + const orgs = parsed.orgs.length > 0 ? parsed.orgs : known; + + console.log( + `🧩 Blocks sync${parsed.apply ? "" : " (dry run: nothing is changed)"}`, + ); + let failed = false; + let pending = 0; + for (const org of orgs) { + let stale: BlocksStaleResource[]; + try { + stale = await blocksStaleRead(rootDir, org); + } catch (error) { + console.error( + `\n${org}\n ❌ ${error instanceof Error ? error.message : error}`, + ); + failed = true; + continue; + } + if (stale.length === 0) continue; + console.log(`\n${org}`); + for (const resource of stale) + console.log( + ` ${relative(join("resources", org), resource.path)} (block changed: ${resource.blocks.join(", ")})`, + ); + if (!parsed.apply) { + pending += stale.length; + continue; + } + console.log(`\n🚀 Applying ${stale.length} resource(s) to ${org}...\n`); + const succeeded = deploy( + org, + stale.map((resource) => resource.path), + ); + const left = await blocksStaleRead(rootDir, org); + if (succeeded && left.length === 0) continue; + failed = true; + console.error( + `\n❌ ${org}: ${left.length} resource(s) still run the old block text:`, + ); + for (const resource of left) console.error(` - ${resource.path}`); + console.error( + " The output above says why: a resource edited in the dashboard is refused until that edit is resolved.", + ); + } + + if (parsed.apply) { + if (!failed) console.log("\n✅ Every resource is in sync with its blocks."); + return failed ? 1 : 0; + } + if (pending === 0 && !failed) + console.log("\n✅ Every resource is in sync with its blocks."); + if (pending > 0) + console.log( + `\n${pending} resource(s) still run the old block text. To push them: npm run sync-blocks -- --apply`, + ); + return failed ? 1 : 0; +} + +const isMainModule = + resolve(process.argv[1] ?? "") === resolve(fileURLToPath(import.meta.url)); +if (isMainModule) { + syncBlocksRun().then( + (code) => process.exit(code), + (error: unknown) => { + console.error( + `❌ Blocks sync failed: ${error instanceof Error ? error.message : String(error)}`, + ); + process.exit(1); + }, + ); +} diff --git a/src/types.ts b/src/types.ts index 6b8cd1c..61e3fad 100644 --- a/src/types.ts +++ b/src/types.ts @@ -42,6 +42,9 @@ export interface ResourceFile> { resourceId: string; // Path relative to resource type dir (e.g., "support/intake" or just "intake") filePath: string; data: T; + // Block file → content hash for each block the file resolved to (see + // blocks.ts). Absent or empty when the file uses none. + blocks?: Record; } export interface VapiResponse { diff --git a/src/validate.ts b/src/validate.ts index 10fb3f4..2d6a7eb 100644 --- a/src/validate.ts +++ b/src/validate.ts @@ -473,6 +473,64 @@ export function validateNoIgnoredReferences( return findings; } +// ───────────────────────────────────────────────────────────────────────────── +// Check 7: Sharing syntax that silently does nothing +// +// Two ways of sharing content look right in a file and are dropped without +// an error. The engine does not process YAML merge keys, so "<<" goes to the +// API as a field of that name. And Vapi's call-time Liquid has its file tags +// switched off, so "include", "render" and "layout" render as nothing on a +// call. Blocks ("$include", "@include") are the supported way to do both. +// ───────────────────────────────────────────────────────────────────────────── + +const LIQUID_FILE_TAG = /\{%-?\s*(include|render|layout)\b/; + +function checkIgnoredSharing(resources: LoadedResources): ValidationFinding[] { + const findings: ValidationFinding[] = []; + for (const type of Object.keys(resources) as ResourceType[]) { + for (const resource of resources[type]) { + const visit = (value: unknown, path: string): void => { + if (Array.isArray(value)) { + value.forEach((item, index) => visit(item, `${path}[${index}]`)); + return; + } + if (typeof value !== "object" || value === null) return; + const mapping = value as Record; + if ("<<" in mapping) + findings.push({ + severity: "warn", + type, + resourceId: resource.resourceId, + rule: "yaml-merge-key", + fieldPath: path || undefined, + message: + 'YAML merge key ("<<") is not processed: it is sent to Vapi as a field named "<<". ' + + 'Share config with a config block ("$include") instead — see docs/guides/blocks.md', + }); + const tag = + mapping.role === "system" && typeof mapping.content === "string" + ? mapping.content.match(LIQUID_FILE_TAG)?.[1] + : undefined; + if (tag) + findings.push({ + severity: "warn", + type, + resourceId: resource.resourceId, + rule: "liquid-file-tag", + fieldPath: path || undefined, + message: + `system prompt uses the Liquid "${tag}" tag, which Vapi renders as nothing on a call. ` + + 'Share prompt text with a prompt block ("@include ") instead — see docs/guides/blocks.md', + }); + for (const [key, child] of Object.entries(mapping)) + visit(child, path ? `${path}.${key}` : key); + }; + visit(resource.data, ""); + } + } + return findings; +} + // ───────────────────────────────────────────────────────────────────────────── // Public entry: run all checks // ───────────────────────────────────────────────────────────────────────────── @@ -486,6 +544,7 @@ export function validateResources( ...checkPromptDuplications(resources), ...checkMaxTokensFloor(resources), ...checkVoiceSchemas(resources), + ...checkIgnoredSharing(resources), ]; } diff --git a/tests/blocks-helpers.ts b/tests/blocks-helpers.ts new file mode 100644 index 0000000..4d7b472 --- /dev/null +++ b/tests/blocks-helpers.ts @@ -0,0 +1,186 @@ +// Shared by the blocks tests: a throwaway repository with the engine's own +// src/ in it, and a Vapi API stub on a worker thread (spawnSync parks this +// thread's event loop, so the stub can't live here). + +import { spawnSync } from "node:child_process"; +import { + cpSync, + mkdirSync, + mkdtempSync, + symlinkSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { Worker } from "node:worker_threads"; + +const REPO_ROOT = fileURLToPath(new URL("..", import.meta.url)); + +export function filesWrite(root: string, files: Record): void { + for (const [path, content] of Object.entries(files)) { + const full = join(root, path); + mkdirSync(dirname(full), { recursive: true }); + writeFileSync(full, content); + } +} + +// A repository root holding only the given files. +export function repoCreate(files: Record): string { + const root = mkdtempSync(join(tmpdir(), "vapi-blocks-")); + filesWrite(root, files); + return root; +} + +// The same, plus what a spawned CLI needs to run from it. +export function engineRepoCreate(files: Record): string { + const root = repoCreate(files); + cpSync(join(REPO_ROOT, "src"), join(root, "src"), { recursive: true }); + cpSync(join(REPO_ROOT, "package.json"), join(root, "package.json")); + symlinkSync( + join(REPO_ROOT, "node_modules"), + join(root, "node_modules"), + "dir", + ); + return root; +} + +export function stateFile(assistants: Record): string { + return JSON.stringify( + { + // Non-empty, so push doesn't treat the state as uninitialized and + // bootstrap-pull before the drift check. + credentials: { "unused-credential": { uuid: "cred-uuid-unused" } }, + assistants: Object.fromEntries( + Object.entries(assistants).map(([id, uuid]) => [id, { uuid }]), + ), + structuredOutputs: {}, + tools: {}, + squads: {}, + personalities: {}, + scenarios: {}, + simulations: {}, + simulationSuites: {}, + evals: {}, + }, + null, + 2, + ); +} + +// An assistant file and the payload the platform returns for it. The two +// hash the same when the file's expanded body equals `prompt`. +export function assistantFile(name: string, body: string): string { + return `--- +model: + provider: openai + model: gpt-4o +name: ${name} +voice: + provider: 11labs + voiceId: burt +--- + +${body} +`; +} + +export function assistantPayload(uuid: string, name: string, prompt: string) { + return { + id: uuid, + orgId: "org-test", + name, + model: { + provider: "openai", + model: "gpt-4o", + messages: [{ role: "system", content: prompt }], + }, + voice: { provider: "11labs", voiceId: "burt" }, + }; +} + +export interface StubRoute { + method: string; + pathStartsWith: string; + body: unknown; +} + +export interface Stub { + port: number; + // Every request served so far, as "METHOD /path". + requests: () => Promise; + stop: () => Promise; +} + +// The first matching route answers; anything else gets an empty list. +export function stubStart(routes: StubRoute[]): Promise { + return new Promise((resolveStart, rejectStart) => { + const source = ` + const http = require('node:http'); + const { parentPort, workerData } = require('node:worker_threads'); + const requests = []; + const server = http.createServer((req, res) => { + const method = (req.method || 'GET').toUpperCase(); + const url = req.url || ''; + requests.push(method + ' ' + url); + const match = workerData.routes.find( + (r) => r.method === method && url.startsWith(r.pathStartsWith), + ); + res.statusCode = 200; + res.setHeader('Content-Type', 'application/json'); + res.end(JSON.stringify(match ? match.body : [])); + }); + server.listen(0, '127.0.0.1', () => { + parentPort.postMessage({ type: 'listening', port: server.address().port }); + }); + parentPort.on('message', (msg) => { + if (msg.type === 'requests') parentPort.postMessage({ type: 'requests', requests }); + if (msg.type === 'shutdown') server.close(() => process.exit(0)); + }); + `; + const worker = new Worker(source, { eval: true, workerData: { routes } }); + worker.once("error", rejectStart); + worker.once("message", (message: { port: number }) => { + resolveStart({ + port: message.port, + requests: () => + new Promise((resolveRequests) => { + worker.once("message", (reply: { requests: string[] }) => + resolveRequests(reply.requests), + ); + worker.postMessage({ type: "requests" }); + }), + stop: () => + new Promise((resolveStop) => { + worker.once("exit", () => resolveStop()); + worker.postMessage({ type: "shutdown" }); + setTimeout(() => void worker.terminate(), 1000); + }), + }); + }); + }); +} + +// Run one of the engine's CLIs from `root` against the stub. +export function cliRun( + root: string, + script: string, + args: string[], + port: number, +): { code: number | null; output: string } { + const result = spawnSync( + process.execPath, + ["--import", "tsx", script, ...args], + { + cwd: root, + env: { + ...process.env, + VAPI_TOKEN: "fake-token-not-used", + VAPI_BASE_URL: `http://127.0.0.1:${port}`, + }, + encoding: "utf8", + timeout: 60_000, + }, + ); + return { code: result.status, output: `${result.stdout}\n${result.stderr}` }; +} diff --git a/tests/blocks-promotion.test.ts b/tests/blocks-promotion.test.ts new file mode 100644 index 0000000..13565c3 --- /dev/null +++ b/tests/blocks-promotion.test.ts @@ -0,0 +1,192 @@ +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; +import { type BlockUsage, blocksRecordUpdate } from "../src/blocks.ts"; +import { checksConfigParse } from "../src/check-config.ts"; +import { checkAffectedBy } from "../src/check-select.ts"; +import { + promotionPlanApply, + promotionPlanBuild, + promotionStateParse, +} from "../src/promotion.ts"; +import { FOLDER_MAP, orgResourcesRead } from "../src/resource-parse.ts"; +import { assistantFile, filesWrite } from "./blocks-helpers.ts"; + +// Promotion compares file text between two orgs. A block edit changes no +// file, so without the blocks record it would never reach the next org. + +// promote-cmd.ts binds its root at import, so one fixture serves the file. +const ROOT = mkdtempSync(join(tmpdir(), "blocks-promotion-")); +process.env.VAPI_GITOPS_ROOT = ROOT; +const { promotionCommandRun } = await import("../src/promote-cmd.ts"); + +const TRIAGE = assistantFile("Triage", "You triage.\n\n@include red-flags"); +const STATE = promotionStateParse("{}"); + +async function recordSeed(org: string): Promise { + const usage = new Map(); + for (const resource of (await orgResourcesRead(ROOT, org)).values()) + usage.set(`${FOLDER_MAP[resource.type]}/${resource.id}`, resource.blocks); + blocksRecordUpdate(join(ROOT, "resources", org), usage, usage.keys()); +} + +// dev and prod hold the same file, and prod was last pushed with the block +// as it stands. +async function fixtureReset(): Promise { + rmSync(ROOT, { recursive: true, force: true }); + filesWrite(ROOT, { + "promotion.yml": + "version: 1\norgs:\n dev: {}\n prod: {}\npipelines:\n release:\n orgs: [dev, prod]\n resources: ['**/*']\n", + "templates/blocks/red-flags.md": "Call 999.\n", + "resources/dev/assistants/triage.md": TRIAGE, + "resources/dev/assistants/plain.md": assistantFile("Plain", "Hello."), + "resources/prod/assistants/triage.md": TRIAGE, + "resources/prod/assistants/plain.md": assistantFile("Plain", "Hello."), + ".vapi-state.dev.json": "{}\n", + ".vapi-state.prod.json": "{}\n", + ".gitignore": "tmp/\n.env.*\n", + }); + await recordSeed("prod"); +} + +function plan() { + return promotionPlanBuild({ + rootDir: ROOT, + source: "dev", + target: "prod", + patterns: ["**/*"], + sourceState: STATE, + targetState: STATE, + }); +} + +test("a block-only change is planned as a sync of the target resources that use the block", async () => { + await fixtureReset(); + assert.deepEqual((await plan()).changes, []); + + filesWrite(ROOT, { "templates/blocks/red-flags.md": "Call 999 now.\n" }); + const changed = await plan(); + assert.deepEqual(changed.changes, [ + { + kind: "sync", + path: "assistants/triage.md", + blocks: ["blocks/red-flags.md"], + }, + ]); + + // A sync entry carries no content: applying the plan must not touch the file. + await promotionPlanApply(changed); + assert.equal( + readFileSync(join(ROOT, "resources/prod/assistants/triage.md"), "utf8"), + TRIAGE, + ); +}); + +test("a target with its own block is not synced for a shared edit, and a block added to it is", async () => { + await fixtureReset(); + filesWrite(ROOT, { + "resources/prod/templates/blocks/red-flags.md": "Prod's wording.\n", + }); + assert.deepEqual( + (await plan()).changes.map((change) => change.kind), + ["sync"], + ); + await recordSeed("prod"); + + filesWrite(ROOT, { "templates/blocks/red-flags.md": "Call 999 now.\n" }); + assert.deepEqual((await plan()).changes, []); +}); + +test("a file the target can't expand stops the plan and names the target", async () => { + await fixtureReset(); + filesWrite(ROOT, { + "resources/dev/templates/blocks/dev-only.md": "Only dev has this.\n", + "resources/dev/assistants/triage.md": assistantFile( + "Triage", + "@include dev-only", + ), + "resources/prod/assistants/triage.md": assistantFile( + "Triage", + "@include dev-only", + ), + }); + await assert.rejects( + plan(), + /assistants\/triage\.md can't be expanded for prod: Unknown prompt block "dev-only"/, + ); +}); + +test("promote --apply deploys the synced files to the target", async () => { + await fixtureReset(); + const git = (...args: string[]) => + execFileSync( + "git", + ["-c", "user.name=t", "-c", "user.email=t@example.com", ...args], + { cwd: ROOT, encoding: "utf8" }, + ); + git("init", "-q", "-b", "main"); + git("add", "-A"); + git("commit", "-qm", "base"); + + const applies: Array<[string, string[]]> = []; + const lines: string[] = []; + const log = console.log; + console.log = (line: string) => lines.push(line); + const promote = (args: string[]) => { + // The command clears this once it has read it. + process.env.VAPI_PROMOTION_TOKENS = JSON.stringify({ dev: "t", prod: "t" }); + return promotionCommandRun(args, { + childRun: (script, org, _connection, childArgs) => { + if (script === "src/apply.ts") applies.push([org, childArgs]); + }, + }); + }; + try { + await promote(["--all", "--apply"]); + assert.deepEqual(applies, []); + + filesWrite(ROOT, { "templates/blocks/red-flags.md": "Call 999 now.\n" }); + await promote(["--all", "--apply"]); + } finally { + console.log = log; + } + assert.deepEqual(applies, [ + [ + "prod", + [ + "--force", + "--allow-new-files", + "--resolve=ours", + "resources/prod/assistants/triage.md", + ], + ], + ]); + assert.ok( + lines.includes( + " sync assistants/triage.md (block changed: blocks/red-flags.md)", + ), + lines.join("\n"), + ); +}); + +test("a shared block edit selects every PR check; an org's own block selects that org's", () => { + const { core, other } = checksConfigParse( + "version: 1\nchecks:\n core:\n org: acme\n targets: [squads/main]\n suites: [core]\n other:\n org: other\n targets: [squads/main]\n suites: [core]\n", + ).checks; + const shared = "templates/blocks/red-flags.md"; + const own = "resources/acme/templates/config/voice.yml"; + assert.deepEqual( + [ + checkAffectedBy(core!, [shared]), + checkAffectedBy(other!, [shared]), + checkAffectedBy(core!, [own]), + checkAffectedBy(other!, [own]), + ], + [shared, shared, own, undefined], + ); +}); + +test.after(() => rmSync(ROOT, { recursive: true, force: true })); diff --git a/tests/blocks-pull.test.ts b/tests/blocks-pull.test.ts new file mode 100644 index 0000000..223d884 --- /dev/null +++ b/tests/blocks-pull.test.ts @@ -0,0 +1,188 @@ +import assert from "node:assert/strict"; +import { existsSync, readdirSync, readFileSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import test from "node:test"; +import { + assistantFile, + assistantPayload, + cliRun, + engineRepoCreate, + filesWrite, + stateFile, + stubStart, +} from "./blocks-helpers.ts"; + +// Pull writes the platform's version of a resource over the local file in +// several situations: when the two already agree (it rewrites the file in +// canonical form), when only the dashboard changed, under --force, and under +// --resolve=theirs. The platform only ever holds the expanded text, so each +// of those would replace a marker with a copy of the block. None may touch a +// file that uses blocks. + +const ORG = "test-blocks"; +const TRIAGE = "aaaaaaaa-1111-1111-1111-111111111111"; +const PLAIN = "bbbbbbbb-2222-2222-2222-222222222222"; + +const BLOCK = "Call 999 for chest pain at {{practice_name}}."; +const TRIAGE_FILE = assistantFile( + "Triage", + "You are the triage agent.\n\n@include red-flags", +); +const TRIAGE_EXPANDED = `You are the triage agent.\n\n${BLOCK}`; +const PLAIN_FILE = assistantFile("Plain", "No blocks here."); + +const ASSISTANTS = `resources/${ORG}/assistants`; + +function repo(): string { + return engineRepoCreate({ + "templates/blocks/red-flags.md": `${BLOCK}\n`, + [`${ASSISTANTS}/triage.md`]: TRIAGE_FILE, + [`${ASSISTANTS}/plain.md`]: PLAIN_FILE, + [`.vapi-state.${ORG}.json`]: stateFile({ triage: TRIAGE, plain: PLAIN }), + }); +} + +// Run pull against a dashboard holding these two prompts. +async function pull( + root: string, + prompts: { triage: string; plain: string }, + args: string[] = [], +): Promise { + const stub = await stubStart([ + { + method: "GET", + pathStartsWith: "/assistant", + body: [ + assistantPayload(TRIAGE, "Triage", prompts.triage), + assistantPayload(PLAIN, "Plain", prompts.plain), + ], + }, + ]); + try { + const run = cliRun(root, "src/pull.ts", [ORG, ...args], stub.port); + assert.equal(run.code, 0, run.output); + return run.output; + } finally { + await stub.stop(); + } +} + +function read(root: string, path: string): string { + return readFileSync(join(root, path), "utf8"); +} + +function baseline(root: string, uuid: string): string | undefined { + const path = join(root, ".vapi-state-hash", ORG, uuid); + return existsSync(path) ? readFileSync(path, "utf8") : undefined; +} + +// The dashboard copies pull left beside triage.md. +function backups(root: string): string[] { + return readdirSync(join(root, ASSISTANTS)).filter( + (name) => name.startsWith("triage.") && name.includes(".bkp."), + ); +} + +const IN_SYNC = { triage: TRIAGE_EXPANDED, plain: "No blocks here." }; +const DASHBOARD_EDITED = { + triage: `${TRIAGE_EXPANDED}\nAdded in the dashboard.`, + plain: "Edited in the dashboard.", +}; + +test("pull leaves a file that uses blocks as written when it matches the dashboard", async () => { + const root = repo(); + try { + const output = await pull(root, IN_SYNC); + assert.equal(read(root, `${ASSISTANTS}/triage.md`), TRIAGE_FILE); + assert.match(output, /triage \(uses blocks, matches the dashboard/); + // In sync is recorded, so a later dashboard edit is seen as drift. + assert.ok(baseline(root, TRIAGE)); + assert.deepEqual(backups(root), []); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test("pull keeps the marker when only the dashboard changed, and saves the dashboard version beside it", async () => { + const root = repo(); + try { + await pull(root, IN_SYNC); + const seeded = baseline(root, TRIAGE); + + const output = await pull(root, DASHBOARD_EDITED); + assert.equal(read(root, `${ASSISTANTS}/triage.md`), TRIAGE_FILE); + assert.match( + output, + /triage: the dashboard version differs, but the file uses blocks, so it was not overwritten/, + ); + const [copy] = backups(root); + assert.ok(copy, output); + assert.match( + read(root, `${ASSISTANTS}/${copy}`), + /Added in the dashboard\./, + ); + // The edit is unresolved, so the baseline must not move: the next pull + // and the push drift check still see it. + assert.equal(baseline(root, TRIAGE), seeded); + + // The same pull does sync an ordinary file down, as before. + assert.match( + read(root, `${ASSISTANTS}/plain.md`), + /Edited in the dashboard\./, + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test("pull --force overwrites ordinary files but not one that uses blocks", async () => { + const root = repo(); + try { + const output = await pull(root, DASHBOARD_EDITED, ["--force"]); + assert.equal(read(root, `${ASSISTANTS}/triage.md`), TRIAGE_FILE); + assert.match(output, /triage: the dashboard version differs/); + assert.equal(backups(root).length, 1); + assert.match( + read(root, `${ASSISTANTS}/plain.md`), + /Edited in the dashboard\./, + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test("pull --resolve=theirs does not overwrite a conflicted file that uses blocks", async () => { + const root = repo(); + try { + await pull(root, IN_SYNC); + const seeded = baseline(root, TRIAGE); + const edited = assistantFile( + "Triage", + "You are the triage agent. Edited locally.\n\n@include red-flags", + ); + filesWrite(root, { [`${ASSISTANTS}/triage.md`]: edited }); + + const output = await pull(root, DASHBOARD_EDITED, ["--resolve=theirs"]); + assert.equal(read(root, `${ASSISTANTS}/triage.md`), edited); + assert.match(output, /triage: the dashboard version differs/); + assert.equal(backups(root).length, 1); + assert.equal(baseline(root, TRIAGE), seeded); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test("pull keeps a block edit that has not been pushed yet", async () => { + const root = repo(); + try { + await pull(root, IN_SYNC); + filesWrite(root, { "templates/blocks/red-flags.md": "New wording.\n" }); + + const output = await pull(root, IN_SYNC); + assert.equal(read(root, `${ASSISTANTS}/triage.md`), TRIAGE_FILE); + assert.match(output, /triage \(local ahead of dashboard\)/); + assert.deepEqual(backups(root), []); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/tests/blocks-sync.test.ts b/tests/blocks-sync.test.ts new file mode 100644 index 0000000..8847fde --- /dev/null +++ b/tests/blocks-sync.test.ts @@ -0,0 +1,313 @@ +import assert from "node:assert/strict"; +import { readFileSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import test from "node:test"; +import { + type BlockUsage, + blocksRecordRead, + blocksRecordUpdate, +} from "../src/blocks.ts"; +import { FOLDER_MAP, orgResourcesRead } from "../src/resource-parse.ts"; +import { syncBlocksRun } from "../src/sync-blocks-cmd.ts"; +import { + assistantFile, + assistantPayload, + cliRun, + engineRepoCreate, + filesWrite, + repoCreate, + stateFile, + stubStart, +} from "./blocks-helpers.ts"; + +// A block edit changes no resource file, so nothing in git says which +// resources now run old text. sync-blocks finds them from the blocks record +// and pushes them through apply. + +const TRIAGE = assistantFile("Triage", "You triage.\n\n@include red-flags"); +const PHARMACY = assistantFile("Pharmacy", "Repeats.\n\n@include red-flags"); +const SQUAD = + "name: Front desk\nmembers:\n - assistantId: triage\nmembersOverrides:\n voice:\n $include: voice\n"; + +// The blocks each resource of an org uses right now. +async function usageRead( + root: string, + org: string, +): Promise> { + const usage = new Map(); + for (const resource of (await orgResourcesRead(root, org)).values()) + usage.set(`${FOLDER_MAP[resource.type]}/${resource.id}`, resource.blocks); + return usage; +} + +// Mark every resource of an org as pushed with the blocks it uses right now. +async function recordSeed(root: string, org: string): Promise { + const usage = await usageRead(root, org); + blocksRecordUpdate(join(root, "resources", org), usage, usage.keys()); +} + +async function threeOrgs(): Promise { + const root = repoCreate({ + "templates/blocks/red-flags.md": "Call 999.\n", + "templates/config/voice.yml": "provider: 11labs\nvoiceId: burt\n", + "resources/alpha/assistants/triage.md": TRIAGE, + "resources/alpha/assistants/pharmacy.md": PHARMACY, + "resources/alpha/assistants/plain.md": assistantFile("Plain", "Hello."), + "resources/alpha/squads/front-desk.yml": SQUAD, + // Beta has its own wording, so a shared edit does not reach it. + "resources/beta/assistants/triage.md": TRIAGE, + "resources/beta/templates/blocks/red-flags.md": "Beta's wording.\n", + "resources/gamma/assistants/triage.md": TRIAGE, + }); + for (const org of ["alpha", "beta", "gamma"]) await recordSeed(root, org); + return root; +} + +async function run( + root: string, + args: string[], + applyRun: (org: string, paths: string[]) => boolean, +): Promise<{ code: number; output: string }> { + const lines: string[] = []; + const log = console.log; + const error = console.error; + console.log = (line: string) => lines.push(line); + console.error = (line: string) => lines.push(line); + try { + const code = await syncBlocksRun(args, { rootDir: root, applyRun }); + return { code, output: lines.join("\n") }; + } finally { + console.log = log; + console.error = error; + } +} + +function neverApply(): boolean { + throw new Error("a dry run must not apply"); +} + +test("the dry run lists, across every org, the resources a block edit reached", async () => { + const root = await threeOrgs(); + try { + const clean = await run(root, [], neverApply); + assert.equal(clean.code, 0); + assert.match(clean.output, /Every resource is in sync with its blocks/); + + filesWrite(root, { + "templates/blocks/red-flags.md": "Call 999 now.\n", + "templates/config/voice.yml": "provider: 11labs\nvoiceId: sarah\n", + }); + const dry = await run(root, [], neverApply); + assert.equal(dry.code, 0); + assert.deepEqual(dry.output.split("\n"), [ + "🧩 Blocks sync (dry run: nothing is changed)", + "", + "alpha", + " assistants/pharmacy.md (block changed: blocks/red-flags.md)", + " assistants/triage.md (block changed: blocks/red-flags.md)", + " squads/front-desk.yml (block changed: config/voice.yml)", + "", + "gamma", + " assistants/triage.md (block changed: blocks/red-flags.md)", + "", + "4 resource(s) still run the old block text. To push them: npm run sync-blocks -- --apply", + ]); + + const scoped = await run(root, ["--org", "gamma"], neverApply); + assert.doesNotMatch(scoped.output, /alpha/); + assert.match(scoped.output, /1 resource\(s\) still run the old block text/); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test("--apply deploys each org's affected files and fails while any is left behind", async () => { + const root = await threeOrgs(); + try { + filesWrite(root, { "templates/blocks/red-flags.md": "Call 999 now.\n" }); + const calls: Array<[string, string[]]> = []; + + // An apply that pushes nothing, as when every resource is refused. + const refused = await run(root, ["--apply"], (org, paths) => { + calls.push([org, paths]); + return true; + }); + assert.deepEqual(calls, [ + [ + "alpha", + [ + "resources/alpha/assistants/pharmacy.md", + "resources/alpha/assistants/triage.md", + ], + ], + ["gamma", ["resources/gamma/assistants/triage.md"]], + ]); + assert.equal(refused.code, 1); + assert.match( + refused.output, + /alpha: 2 resource\(s\) still run the old block text/, + ); + + // An apply that pushes them, which is when push updates the record. + const current = { + alpha: await usageRead(root, "alpha"), + gamma: await usageRead(root, "gamma"), + }; + const pushed = await run(root, ["--apply"], (org) => { + const usage = current[org as "alpha" | "gamma"]; + blocksRecordUpdate(join(root, "resources", org), usage, usage.keys()); + return true; + }); + assert.equal(pushed.code, 0, pushed.output); + assert.match(pushed.output, /Every resource is in sync with its blocks/); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test("a broken org is reported and the others are still listed", async () => { + const root = await threeOrgs(); + try { + filesWrite(root, { + "templates/blocks/red-flags.md": "Call 999 now.\n", + "resources/alpha/assistants/triage.md": assistantFile( + "Triage", + "@include no-such-block", + ), + }); + const result = await run(root, [], neverApply); + assert.equal(result.code, 1); + assert.match(result.output, /alpha\n {2}❌ .*Unknown prompt block/); + assert.match(result.output, /gamma\n {2}assistants\/triage\.md/); + await assert.rejects( + run(root, ["--org", "nope"], neverApply), + /No resources\/nope\/ folder/, + ); + await assert.rejects(run(root, ["--push"], neverApply), /Unrecognized/); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +// End to end, through the real apply → validate → pull → push path against +// a stub API: the resource whose dashboard copy is untouched is updated, and +// the one that was edited in the dashboard is refused by the drift check. +test("--apply pushes through the drift check: an untouched resource is updated, a dashboard-edited one is refused", async () => { + const ORG = "alpha"; + const TRIAGE_ID = "aaaaaaaa-1111-1111-1111-111111111111"; + const PHARMACY_ID = "bbbbbbbb-2222-2222-2222-222222222222"; + const assistants = `resources/${ORG}/assistants`; + const root = engineRepoCreate({ + "templates/blocks/red-flags.md": "Call 999.\n", + [`${assistants}/triage.md`]: TRIAGE, + [`${assistants}/pharmacy.md`]: PHARMACY, + [`.vapi-state.${ORG}.json`]: stateFile({ + triage: TRIAGE_ID, + pharmacy: PHARMACY_ID, + }), + }); + const triage = (block: string) => + assistantPayload(TRIAGE_ID, "Triage", `You triage.\n\n${block}`); + const pharmacy = (block: string) => + assistantPayload(PHARMACY_ID, "Pharmacy", `Repeats.\n\n${block}`); + try { + // Both in sync with the old block: a pull records that as the baseline. + const before = await stubStart([ + { + method: "GET", + pathStartsWith: "/assistant", + body: [triage("Call 999."), pharmacy("Call 999.")], + }, + ]); + try { + const seeded = cliRun(root, "src/pull.ts", [ORG], before.port); + assert.equal(seeded.code, 0, seeded.output); + } finally { + await before.stop(); + } + await recordSeed(root, ORG); + const recorded = blocksRecordRead(join(root, "resources", ORG)); + + // The block changes, and someone edits pharmacy in the dashboard. + filesWrite(root, { "templates/blocks/red-flags.md": "Call 999 now.\n" }); + const edited = pharmacy("Call 999. Edited in the dashboard."); + const stub = await stubStart([ + // An org with no credentials makes push re-pull its state first, which + // resets every baseline to the dashboard's current content. + { + method: "GET", + pathStartsWith: "/credential", + body: [{ id: "cred-uuid-unused", name: "unused-credential" }], + }, + { + method: "GET", + pathStartsWith: `/assistant/${TRIAGE_ID}`, + body: triage("Call 999."), + }, + { + method: "PATCH", + pathStartsWith: `/assistant/${TRIAGE_ID}`, + body: triage("Call 999 now."), + }, + { + method: "GET", + pathStartsWith: `/assistant/${PHARMACY_ID}`, + body: edited, + }, + { + method: "GET", + pathStartsWith: "/assistant", + body: [triage("Call 999."), edited], + }, + ]); + try { + const result = cliRun( + root, + "src/sync-blocks-cmd.ts", + ["--org", ORG, "--apply"], + stub.port, + ); + const patched = (await stub.requests()).filter((request) => + request.startsWith("PATCH /assistant/"), + ); + assert.deepEqual( + { + code: result.code, + triagePatched: patched.some((r) => r.includes(TRIAGE_ID)), + pharmacyPatched: patched.some((r) => r.includes(PHARMACY_ID)), + refusal: /drift detected on assistant pharmacy/.test(result.output), + leftBehind: + /1 resource\(s\) still run the old block text:\n\s+- resources\/alpha\/assistants\/pharmacy\.md/.test( + result.output, + ), + }, + { + code: 1, + triagePatched: true, + pharmacyPatched: false, + refusal: true, + leftBehind: true, + }, + result.output, + ); + } finally { + await stub.stop(); + } + + // Only the pushed resource is recorded with the new block. + const after = blocksRecordRead(join(root, "resources", ORG)); + assert.notDeepEqual(after["assistants/triage"], recorded["assistants/triage"]); + assert.deepEqual( + after["assistants/pharmacy"], + recorded["assistants/pharmacy"], + ); + // And neither file lost its marker on the way through pull. + assert.equal(readFileSync(join(root, assistants, "triage.md"), "utf8"), TRIAGE); + assert.equal( + readFileSync(join(root, assistants, "pharmacy.md"), "utf8"), + PHARMACY, + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/tests/blocks-validate.test.ts b/tests/blocks-validate.test.ts new file mode 100644 index 0000000..d4f55c5 --- /dev/null +++ b/tests/blocks-validate.test.ts @@ -0,0 +1,71 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +// validate.ts reaches config.ts, which reads argv and the token at import. +process.argv = ["node", "test", "test-fixture-org"]; +process.env.VAPI_TOKEN = process.env.VAPI_TOKEN || "test-token-not-used"; + +const { validateResources } = await import("../src/validate.ts"); + +import type { LoadedResources } from "../src/types.ts"; + +// Two ways of sharing content that look right in a file and do nothing: +// validate points at blocks instead. + +function resources(partial: Partial): LoadedResources { + return { + tools: [], + structuredOutputs: [], + assistants: [], + squads: [], + personalities: [], + scenarios: [], + simulations: [], + simulationSuites: [], + evals: [], + ...partial, + }; +} + +test("a YAML merge key is reported, because the engine sends it as a field", () => { + const findings = validateResources( + resources({ + squads: [ + { + resourceId: "front-desk", + filePath: "/fake/front-desk.yml", + data: { + name: "Front Desk", + members: [ + { assistantId: "a", assistantOverrides: { "<<": { x: 1 } } }, + ], + }, + }, + ], + }), + ); + assert.deepEqual( + findings.map((f) => [f.severity, f.type, f.rule, f.fieldPath]), + [["warn", "squads", "yaml-merge-key", "members[0].assistantOverrides"]], + ); + assert.match(findings[0]!.message, /\$include/); +}); + +test("a Liquid file tag in a system prompt is reported, because Vapi renders it as nothing", () => { + const prompt = (content: string) => ({ + resourceId: "a", + filePath: "/fake/a.md", + data: { name: "A", model: { messages: [{ role: "system", content }] } }, + }); + const rules = (content: string) => + validateResources(resources({ assistants: [prompt(content)] })).map( + (f) => f.rule, + ); + assert.deepEqual(rules("Hi.\n{% include 'red-flags' %}"), ["liquid-file-tag"]); + assert.deepEqual(rules("{%- render 'tone' -%}"), ["liquid-file-tag"]); + // Ordinary Liquid, and the block marker, are fine. + assert.deepEqual( + rules("{% if vip %}Hello {{ name }}{% endif %}\n@include red-flags"), + [], + ); +}); diff --git a/tests/blocks.test.ts b/tests/blocks.test.ts new file mode 100644 index 0000000..54b33cc --- /dev/null +++ b/tests/blocks.test.ts @@ -0,0 +1,513 @@ +import assert from "node:assert/strict"; +import { readFileSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import test from "node:test"; +import { + BLOCKS_RECORD_FILE, + blockLayers, + blocksChanged, + blocksExpand, + blocksRecordRead, + blocksRecordUpdate, + blocksUsedIn, +} from "../src/blocks.ts"; +import { + orgResourcesRead, + parseResourceDataFromFile, + resourceFileUsesBlocks, +} from "../src/resource-parse.ts"; +import { filesWrite, repoCreate } from "./blocks-helpers.ts"; + +// Blocks are looked up from an org's folder, so every test builds a small +// repository: templates/ at the root, and resources//templates/. + +function withRepo(files: Record, run: (root: string) => T) { + const root = repoCreate(files); + try { + return run(root); + } finally { + rmSync(root, { recursive: true, force: true }); + } +} + +function expand(root: string, data: unknown, org = "acme") { + return blocksExpand(data, blockLayers(join(root, "resources", org))); +} + +function prompt(content: string) { + return { model: { messages: [{ role: "system", content }] } }; +} + +function promptOf(data: unknown): string { + return (data as ReturnType).model.messages[0]!.content; +} + +test("a prompt block replaces its marker line, and nothing else changes", () => { + withRepo( + { "templates/blocks/red-flags.md": "\nCall 999 for chest pain.\n\n" }, + (root) => { + const { data, usage } = expand( + root, + prompt("You are the triage agent.\n\n@include red-flags\n\nBe brief."), + ); + assert.equal( + promptOf(data), + "You are the triage agent.\n\nCall 999 for chest pain.\n\nBe brief.", + ); + assert.deepEqual(Object.keys(usage), ["blocks/red-flags.md"]); + }, + ); +}); + +test("the org's block wins over the shared block of the same name", () => { + withRepo( + { + "templates/blocks/red-flags.md": "shared wording", + "resources/south/templates/blocks/red-flags.md": "South's wording", + }, + (root) => { + const data = prompt("@include red-flags"); + assert.equal(promptOf(expand(root, data, "acme").data), "shared wording"); + assert.equal( + promptOf(expand(root, data, "south").data), + "South's wording", + ); + }, + ); +}); + +test("an org block that includes its own name extends the shared block", () => { + withRepo( + { + "templates/blocks/red-flags.md": "shared wording", + "resources/south/templates/blocks/red-flags.md": + "@include red-flags\nAlso mention the South walk-in centre.", + }, + (root) => { + const { data, usage } = expand( + root, + prompt("@include red-flags"), + "south", + ); + assert.equal( + promptOf(data), + "shared wording\nAlso mention the South walk-in centre.", + ); + // Both files are behind the one name, so a change to either shows. + const before = usage["blocks/red-flags.md"]; + filesWrite(root, { "templates/blocks/red-flags.md": "new shared" }); + const after = expand(root, prompt("@include red-flags"), "south").usage; + assert.notEqual(after["blocks/red-flags.md"], before); + }, + ); +}); + +test("blocks nest, and every block reached is recorded", () => { + withRepo( + { + "templates/blocks/safety/all.md": + "@include safety/red-flags\n@include tone", + "templates/blocks/safety/red-flags.md": "Call 999.", + "templates/blocks/tone.md": "Stay calm.", + }, + (root) => { + const { data, usage } = expand(root, prompt("@include safety/all")); + assert.equal(promptOf(data), "Call 999.\nStay calm."); + assert.deepEqual(Object.keys(usage), [ + "blocks/safety/all.md", + "blocks/safety/red-flags.md", + "blocks/tone.md", + ]); + }, + ); +}); + +test("a loop of blocks is an error that names the chain", () => { + withRepo( + { + "templates/blocks/a.md": "@include b", + "templates/blocks/b.md": "@include a", + "templates/blocks/self.md": "@include self", + }, + (root) => { + assert.throws( + () => expand(root, prompt("@include a")), + /loop: templates\/blocks\/a\.md → templates\/blocks\/b\.md → templates\/blocks\/a\.md/, + ); + assert.throws(() => expand(root, prompt("@include self")), /loop/); + }, + ); +}); + +test("an unknown block is an error that lists where it was looked for", () => { + withRepo({}, (root) => { + assert.throws( + () => expand(root, prompt("@include red-flags")), + (error: Error) => + error.message.includes('Unknown prompt block "red-flags"') && + error.message.includes( + "resources/acme/templates/blocks/red-flags.md", + ) && + error.message.includes("- templates/blocks/red-flags.md"), + ); + assert.throws( + () => expand(root, { voice: { $include: "voice" } }), + /Unknown config block "voice"[\s\S]*templates\/config\/voice\.yml/, + ); + assert.throws( + () => expand(root, prompt("@include ../secrets")), + /Invalid prompt block name/, + ); + }); +}); + +test("Liquid inside a block reaches the prompt exactly as written", () => { + const block = [ + "Offer to connect them to {{practice_name}} reception.", + "{% if anon_handover %}Do not ask for a name.{% endif %}", + "{{ now | date: '%A' }} {{{raw}}} {%- comment -%}x{%- endcomment -%}", + ].join("\n"); + withRepo({ "templates/blocks/red-flags.md": `${block}\n` }, (root) => { + assert.equal( + promptOf(expand(root, prompt("Hi {{name}}.\n@include red-flags")).data), + `Hi {{name}}.\n${block}`, + ); + }); +}); + +test("only a whole-line marker in a system message is an include", () => { + withRepo({ "templates/blocks/x.md": "X" }, (root) => { + const data = { + firstMessage: "@include x", + model: { + messages: [ + { role: "system", content: "Say @include x aloud.\n @include x " }, + { role: "user", content: "@include x" }, + ], + }, + }; + assert.deepEqual(expand(root, data).data, { + firstMessage: "@include x", + model: { + messages: [ + { role: "system", content: "Say @include x aloud.\nX" }, + { role: "user", content: "@include x" }, + ], + }, + }); + }); +}); + +test("a config block fills a mapping, and the file's own keys win", () => { + withRepo( + { + "templates/config/voice.yml": + "provider: 11labs\nvoiceId: burt\nchunkPlan:\n enabled: true\n minCharacters: 30\n", + }, + (root) => { + const { data, usage } = expand(root, { + name: "Triage", + voice: { + $include: "voice", + voiceId: "sarah", + chunkPlan: { minCharacters: 10 }, + }, + }); + assert.deepEqual(data, { + name: "Triage", + voice: { + provider: "11labs", + voiceId: "sarah", + chunkPlan: { enabled: true, minCharacters: 10 }, + }, + }); + assert.deepEqual(Object.keys(usage), ["config/voice.yml"]); + }, + ); +}); + +test("a list the file sets replaces the block's list whole", () => { + withRepo( + { + "templates/config/model.yml": + "provider: openai\ntoolIds:\n - end-call\n - transfer\n", + }, + (root) => { + assert.deepEqual( + expand(root, { model: { $include: "model", toolIds: ["lookup"] } }) + .data, + { model: { provider: "openai", toolIds: ["lookup"] } }, + ); + }, + ); +}); + +test("config blocks work at the top level, in list items and inside each other", () => { + withRepo( + { + "templates/config/base.yml": + "voice:\n $include: voice\nmodel:\n provider: openai\n messages:\n - role: system\n content: |-\n Shared intro.\n @include tone\n", + "templates/config/voice.yml": "provider: 11labs\nvoiceId: burt\n", + "templates/config/member.yml": + "assistantOverrides:\n voice:\n $include: voice\n", + "templates/blocks/tone.md": "Stay calm, {{caller_name}}.", + }, + (root) => { + const { data, usage } = expand(root, { + $include: "base", + name: "Triage", + members: [{ $include: "member", assistantId: "triage" }], + }); + assert.deepEqual(data, { + name: "Triage", + voice: { provider: "11labs", voiceId: "burt" }, + model: { + provider: "openai", + messages: [ + { + role: "system", + content: "Shared intro.\nStay calm, {{caller_name}}.", + }, + ], + }, + members: [ + { + assistantId: "triage", + assistantOverrides: { + voice: { provider: "11labs", voiceId: "burt" }, + }, + }, + ], + }); + assert.deepEqual(Object.keys(usage), [ + "blocks/tone.md", + "config/base.yml", + "config/member.yml", + "config/voice.yml", + ]); + }, + ); +}); + +test("the org's config block wins, and can extend the shared one", () => { + withRepo( + { + "templates/config/voice.yml": "provider: 11labs\nvoiceId: burt\n", + "resources/south/templates/config/voice.yml": + "$include: voice\nvoiceId: sarah\n", + }, + (root) => { + const data = { voice: { $include: "voice" } }; + assert.deepEqual(expand(root, data, "acme").data, { + voice: { provider: "11labs", voiceId: "burt" }, + }); + assert.deepEqual(expand(root, data, "south").data, { + voice: { provider: "11labs", voiceId: "sarah" }, + }); + }, + ); +}); + +test("a block that is a list can be included on its own", () => { + withRepo( + { "templates/config/tools.yml": "- end-call\n- transfer\n" }, + (root) => { + assert.deepEqual( + expand(root, { model: { toolIds: { $include: "tools" } } }).data, + { model: { toolIds: ["end-call", "transfer"] } }, + ); + assert.throws( + () => expand(root, { model: { $include: "tools", provider: "x" } }), + /Config block "tools" is not a mapping/, + ); + }, + ); +}); + +test("two blocks in one place may not set the same field differently", () => { + withRepo( + { + "templates/config/a.yml": "voice:\n provider: 11labs\n speed: 1\n", + "templates/config/b.yml": "voice:\n provider: 11labs\n speed: 1.2\n", + "templates/config/c.yml": "voice:\n provider: 11labs\n style: 0.4\n", + }, + (root) => { + assert.throws( + () => expand(root, { $include: ["a", "b"] }), + /Config blocks "a" and "b" both set "voice\.speed" to different values/, + ); + // Still refused when the file settles it: the blocks stay ambiguous + // for the next file that includes them. + assert.throws( + () => expand(root, { $include: ["a", "b"], voice: { speed: 1.1 } }), + /both set "voice\.speed"/, + ); + assert.deepEqual(expand(root, { $include: ["a", "c"] }).data, { + voice: { provider: "11labs", speed: 1, style: 0.4 }, + }); + }, + ); +}); + +test("unclear includes are refused", () => { + withRepo({ "templates/config/a.yml": "x: 1\n" }, (root) => { + assert.throws( + () => expand(root, { $include: "a", "<<": { x: 2 } }), + /uses both "\$include" and a YAML merge key/, + ); + assert.throws(() => expand(root, { $include: 3 }), /takes a config block/); + assert.throws(() => expand(root, { $include: [] }), /takes a config block/); + assert.throws( + () => expand(root, { $include: ["a", "a"] }), + /lists the same config block twice/, + ); + }); + withRepo({ "templates/config/empty.yml": "# nothing yet\n" }, (root) => { + assert.throws(() => expand(root, { $include: "empty" }), /is empty/); + }); +}); + +test("a resource that uses no blocks is returned untouched", () => { + withRepo({}, (root) => { + const data = { name: "Plain", model: { messages: [] } }; + const result = expand(root, data); + assert.equal(result.data, data); + assert.deepEqual(result.usage, {}); + assert.equal(blocksUsedIn(data), false); + assert.equal(blocksUsedIn({ members: [{ $include: "m" }] }), true); + assert.equal(blocksUsedIn(prompt("a\n@include b\nc")), true); + }); +}); + +test("every loader expands blocks: Markdown body, YAML assistant, squad", async () => { + const root = repoCreate({ + "templates/blocks/red-flags.md": "Call 999 for {{practice_name}}.\n", + "templates/config/voice.yml": "provider: 11labs\nvoiceId: burt\n", + "resources/acme/assistants/triage.md": + "---\nname: Triage\nvoice:\n $include: voice\n---\n\nYou triage.\n\n@include red-flags\n", + "resources/acme/assistants/pharmacy.yml": + "name: Pharmacy\nmodel:\n messages:\n - role: system\n content: |-\n You handle repeats.\n @include red-flags\n", + "resources/acme/squads/front-desk.yml": + "name: Front desk\nmembers:\n - assistantId: triage\n assistantOverrides:\n voice:\n $include: voice\n voiceId: sarah\n", + }); + try { + const resources = await orgResourcesRead(root, "acme"); + const triage = resources.get("assistants:triage")!; + assert.deepEqual(triage.data, { + name: "Triage", + voice: { provider: "11labs", voiceId: "burt" }, + model: { + messages: [ + { + role: "system", + content: "You triage.\n\nCall 999 for {{practice_name}}.", + }, + ], + }, + }); + assert.deepEqual(Object.keys(triage.blocks), [ + "blocks/red-flags.md", + "config/voice.yml", + ]); + assert.equal( + promptOf(resources.get("assistants:pharmacy")!.data), + "You handle repeats.\nCall 999 for {{practice_name}}.", + ); + assert.deepEqual(resources.get("squads:front-desk")!.data, { + name: "Front desk", + members: [ + { + assistantId: "triage", + assistantOverrides: { + voice: { provider: "11labs", voiceId: "sarah" }, + }, + }, + ], + }); + + // The drift hash is computed from this parse, so it must expand too. + const file = join(root, "resources/acme/assistants/triage.md"); + assert.deepEqual( + parseResourceDataFromFile(file, join(root, "resources/acme")), + triage.data, + ); + assert.throws(() => parseResourceDataFromFile(file), /uses blocks/); + assert.equal(resourceFileUsesBlocks(file), true); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test("a block error names the resource file", async () => { + const root = repoCreate({ + "resources/acme/assistants/triage.md": + "---\nname: Triage\n---\n\n@include missing\n", + }); + try { + await assert.rejects( + orgResourcesRead(root, "acme"), + /Failed to expand blocks in "assistants\/triage\.md": Unknown prompt block "missing"/, + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test("resourceFileUsesBlocks reads the file as written", () => { + withRepo( + { + "a.md": "---\nname: A\n---\n\nNo blocks here, @include is mid-line.\n", + "b.yml": "name: B\nvoice:\n $include: voice\n", + "broken.md": "---\nname: [unclosed\n---\n\n@include red-flags\n", + }, + (root) => { + assert.equal(resourceFileUsesBlocks(join(root, "a.md")), false); + assert.equal(resourceFileUsesBlocks(join(root, "b.yml")), true); + // Unparseable, but it plainly uses a block: pull must leave it alone. + assert.equal(resourceFileUsesBlocks(join(root, "broken.md")), true); + }, + ); +}); + +test("the blocks record marks only pushed resources as in sync", () => { + withRepo({ "resources/acme/.keep": "" }, (root) => { + const dir = join(root, "resources/acme"); + const current = new Map>([ + ["assistants/triage", { "blocks/red-flags.md": "aaaa" }], + ["assistants/pharmacy", { "blocks/red-flags.md": "aaaa" }], + ["assistants/plain", {}], + ]); + blocksRecordUpdate(dir, current, ["assistants/triage", "assistants/plain"]); + assert.deepEqual(blocksRecordRead(dir), { + "assistants/triage": { "blocks/red-flags.md": "aaaa" }, + }); + assert.match( + readFileSync(join(dir, BLOCKS_RECORD_FILE), "utf8"), + /^\{\n {2}"assistants\/triage"/, + ); + + // The block changes; only pharmacy is pushed. Triage keeps its old entry, + // which is what marks it as still running the old text. + const changed = new Map>([ + ["assistants/triage", { "blocks/red-flags.md": "bbbb" }], + ["assistants/pharmacy", { "blocks/red-flags.md": "bbbb" }], + ]); + blocksRecordUpdate(dir, changed, ["assistants/pharmacy"]); + const record = blocksRecordRead(dir); + assert.deepEqual( + blocksChanged(changed.get("assistants/triage")!, record["assistants/triage"]), + ["blocks/red-flags.md"], + ); + assert.deepEqual( + blocksChanged( + changed.get("assistants/pharmacy")!, + record["assistants/pharmacy"], + ), + [], + ); + + // A resource that is gone, or no longer uses blocks, drops out. + blocksRecordUpdate(dir, new Map([["assistants/pharmacy", {}]]), []); + assert.deepEqual(blocksRecordRead(dir), {}); + }); +}); diff --git a/tests/examples.test.ts b/tests/examples.test.ts index 93e015d..804399e 100644 --- a/tests/examples.test.ts +++ b/tests/examples.test.ts @@ -40,7 +40,7 @@ function exampleOrgs(): Array<{ root: string; org: string }> { } function validateRun( - resourcesDir: string, + root: string, org: string, ): { code: number | null; output: string } { const dir = mkdtempSync(join(tmpdir(), "vapi-examples-")); @@ -49,7 +49,14 @@ function validateRun( cpSync(join(REPO, "package.json"), join(dir, "package.json")); symlinkSync(join(REPO, "node_modules"), join(dir, "node_modules"), "dir"); mkdirSync(join(dir, "resources")); - cpSync(resourcesDir, join(dir, "resources", org), { recursive: true }); + cpSync(join(root, "resources", org), join(dir, "resources", org), { + recursive: true, + }); + // An example's shared blocks sit beside its resources/ folder. + if (existsSync(join(root, "templates"))) + cpSync(join(root, "templates"), join(dir, "templates"), { + recursive: true, + }); writeFileSync(join(dir, `.env.${org}`), "VAPI_TOKEN=fake-token-not-used\n"); const result = spawnSync( process.execPath, @@ -64,7 +71,7 @@ function validateRun( test("every example org passes validate", () => { const results = exampleOrgs().map(({ root, org }) => { - const run = validateRun(join(root, "resources", org), org); + const run = validateRun(root, org); return [org, run.code, run.code === 0 ? "" : run.output]; }); assert.deepEqual(