diff --git a/platform/packages/api/src/deploy/upload-worker.ts b/platform/packages/api/src/deploy/upload-worker.ts index d4a481ce3..5cd754645 100644 --- a/platform/packages/api/src/deploy/upload-worker.ts +++ b/platform/packages/api/src/deploy/upload-worker.ts @@ -693,7 +693,9 @@ export async function uploadDeploymentWorker(input: { }, ); input.workerUploaded = true; - await verifyManagedDurableObjectApplicationBindings(context.env, workerName, manifest); + await verifyManagedDurableObjectApplicationBindings(context.env, workerName, manifest, { + deadline: input.deadline, + }); await assertDeploymentCanProceed(context.get('db'), deployment.id, projectId); diff --git a/platform/packages/api/src/provisioning/durable-objects.ts b/platform/packages/api/src/provisioning/durable-objects.ts index ef802ce62..53d4e6bf6 100644 --- a/platform/packages/api/src/provisioning/durable-objects.ts +++ b/platform/packages/api/src/provisioning/durable-objects.ts @@ -1,3 +1,4 @@ +import { setTimeout as delay } from 'node:timers/promises'; import { and, eq, sql } from 'drizzle-orm'; import { createAssetUploadSession, @@ -32,6 +33,7 @@ import { CF_API_BASE, cfFetch, CloudflareApiError } from './cloudflare-api.ts'; import { putDurableSnapshotModule, putDurableSnapshotObject } from './durable-snapshot.ts'; type Env = ApiEnv['Bindings']; +type DispatchWorkerEnv = Pick; type Binding = Record & { type: string; name: string }; type Snapshot = { metadata: Record & { bindings: Array }; @@ -60,7 +62,7 @@ export async function durableObjectWorkerName( return `void-do-${projectId}-${digest.slice(0, 16)}`; } -function url(env: Env, worker: string): string { +function url(env: DispatchWorkerEnv, worker: string): string { return `${CF_API_BASE}/${env.CF_ACCOUNT_ID}/workers/dispatch/namespaces/${namespace(env)}/scripts/${worker}`; } @@ -72,17 +74,25 @@ function marker( return JSON.stringify({ protocol: 1, projectId, dispatchNamespace: namespace(env), className }); } -async function readBindings(env: Env, worker: string): Promise | undefined> { +async function readBindings( + env: DispatchWorkerEnv, + worker: string, + { + confirmWorkerExists = true, + signal, + }: { confirmWorkerExists?: boolean; signal?: AbortSignal } = {}, +): Promise | undefined> { try { const bindings = await cfFetch>( `${url(env, worker)}/bindings`, env.CF_API_TOKEN, + { signal }, ); // Dispatch can return an empty binding list for a script without a Worker. // Confirm absence before provisioning; an existing unmarked Worker must // still fail the ownership check. - if (bindings.length === 0) { - await cfFetch(`${url(env, worker)}/settings`, env.CF_API_TOKEN); + if (confirmWorkerExists && bindings.length === 0) { + await cfFetch(`${url(env, worker)}/settings`, env.CF_API_TOKEN, { signal }); } return bindings; } catch (error) { @@ -350,19 +360,55 @@ export function managedDurableObjectBindings(manifest: DeployManifest): Array } = {}, ): Promise { - if (!manifest.durableObjects?.bindings.length) { + const expectedBindings = managedDurableObjectBindings(manifest); + if (!expectedBindings.length) { return; } - const actual = await readBindings(env, worker); - for (const expected of managedDurableObjectBindings(manifest)) { - const binding = actual?.find((value) => value.name === expected.name); - if (binding?.type !== expected.type || binding.namespace_id !== expected.namespace_id) { + const matchesExpected = (actual: Array | undefined): boolean => + expectedBindings.every((expected) => { + const binding = actual?.find((value) => value.name === expected.name); + return binding?.type === expected.type && binding.namespace_id === expected.namespace_id; + }); + + // Keep queued verification inside the same deadline as the uploads so a + // slow series of reads cannot spend the time reserved for cleanup. + const remaining = deadline === undefined ? undefined : deadline - Date.now(); + const timeout = new Error( + 'The deployment reached its time limit while verifying application Durable Object bindings.', + ); + if (remaining !== undefined && remaining <= 0) { + throw timeout; + } + const controller = new AbortController(); + const timer = + remaining === undefined ? undefined : setTimeout(() => controller.abort(timeout), remaining); + const signal = controller.signal; + const read = () => readBindings(env, worker, { confirmWorkerExists: false, signal }); + try { + let actual = await read(); + for (const wait of delaysMs) { + if (matchesExpected(actual)) { + return; + } + await delay(wait, undefined, { signal }); + actual = await read(); + } + if (!matchesExpected(actual)) { throw new Error('Application custom Durable Object binding could not be verified.'); } + } catch (error) { + signal.throwIfAborted(); + throw error; + } finally { + clearTimeout(timer); } } diff --git a/platform/packages/api/test/integration/managed-durable-objects.test.ts b/platform/packages/api/test/integration/managed-durable-objects.test.ts index 84ca84628..64fe90745 100644 --- a/platform/packages/api/test/integration/managed-durable-objects.test.ts +++ b/platform/packages/api/test/integration/managed-durable-objects.test.ts @@ -526,7 +526,21 @@ describe('managed native Durable Objects', () => { expect(workers.map(currentRelease).sort()).toEqual([first.id, second.id].sort()); }); - it('ends the release uploads of a queued deployment at the run deadline', async () => { + it.each([ + { + phase: 'application binding verification', + elapsed: 600_000, + readsBindings: false, + message: + 'The deployment reached its time limit while verifying application Durable Object bindings.', + }, + { + phase: 'Durable Object host upload', + elapsed: 570_000, + readsBindings: true, + message: WORKER_UPLOAD_OUT_OF_TIME, + }, + ])('enforces the queued deadline during $phase', async ({ elapsed, readsBindings, message }) => { const first = await deploy(); const worker = await durableObjectWorkerName(project.id, binding.className); const queuedEnv = { @@ -547,12 +561,13 @@ describe('managed native Durable Objects', () => { ); await response.body?.cancel(); const id = response.headers.get('x-void-deployment-id')!; - // The application upload ends 9.5 minutes into the run. Too little is left - // to move the host and still move it back, so the host is not touched. + const candidate = `user-${project.slug}-${id}`; const now = Date.now.bind(Date); vi.spyOn(Date, 'now').mockImplementation( - () => now() + (provider.state.hostScripts.has(`user-${project.slug}-${id}`) ? 570_000 : 0), + () => now() + (provider.state.hostScripts.has(candidate) ? elapsed : 0), ); + const fetchMock = vi.fn(globalThis.fetch); + vi.stubGlobal('fetch', fetchMock); const sent = writes.filter((write) => write.name === worker).length; await processDeploymentBatch( @@ -566,8 +581,12 @@ describe('managed native Durable Objects', () => { expect((await getDeployment(env.db, id))?.status).toBe('failed'); expect((await getDeploymentOperation(env.db, id))?.events.at(-1)?.event).toMatchObject({ event: 'error', - message: WORKER_UPLOAD_OUT_OF_TIME, + message, }); + expect( + fetchMock.mock.calls.some(([url]) => String(url).endsWith(`/scripts/${candidate}/bindings`)), + ).toBe(readsBindings); + expect(provider.state.hostScripts.has(candidate)).toBe(false); expect(writes.filter((write) => write.name === worker)).toHaveLength(sent); expect(currentRelease(worker)).toBe(first.id); }); diff --git a/platform/packages/api/test/unit/deploy-durable-objects.test.ts b/platform/packages/api/test/unit/deploy-durable-objects.test.ts index 459bd11a6..b41ff319c 100644 --- a/platform/packages/api/test/unit/deploy-durable-objects.test.ts +++ b/platform/packages/api/test/unit/deploy-durable-objects.test.ts @@ -1,4 +1,5 @@ -import { describe, expect, it } from 'vite-plus/test'; +import { afterEach, describe, expect, it, vi } from 'vite-plus/test'; +import { verifyManagedDurableObjectApplicationBindings } from '../../src/provisioning/durable-objects.ts'; import { buildDurableObjectUploadMigrations, buildLiveUploadBindings, @@ -634,3 +635,111 @@ describe('validateManifestShape sandbox + static/spa rejection', () => { ).toBeNull(); }); }); + +describe('verifyManagedDurableObjectApplicationBindings', () => { + afterEach(() => { + vi.useRealTimers(); + vi.unstubAllGlobals(); + }); + + const env = { + CF_ACCOUNT_ID: 'acc-123', + CF_API_TOKEN: 'tok-123', + DISPATCH_NAMESPACE: 'void-prod', + }; + const manifest: DeployManifest = { + version: 3, + bindings: {}, + durableObjects: { + bindings: [{ bindingName: 'TRACKER', className: 'TrackerDo', storage: 'sqlite' }], + runtime: { protocol: 1, namespaces: { TrackerDo: 'ns-expected' } }, + }, + }; + + it('resolves on first read when bindings match immediately', async () => { + const fetchMock = vi.fn(async () => + Response.json({ + success: true, + errors: [], + result: [ + { name: 'TRACKER', type: 'durable_object_namespace', namespace_id: 'ns-expected' }, + ], + }), + ); + vi.stubGlobal('fetch', fetchMock); + + await expect( + verifyManagedDurableObjectApplicationBindings(env, 'worker-1', manifest), + ).resolves.toBeUndefined(); + expect(fetchMock).toHaveBeenCalledTimes(1); + }); + + it('retries empty application bindings without looking up Worker settings', async () => { + let bindingCalls = 0; + const fetchMock = vi.fn(async (input: RequestInfo | URL) => { + expect(String(input)).toMatch(/\/scripts\/worker-1\/bindings$/); + bindingCalls += 1; + return Response.json({ + success: true, + errors: [], + result: + bindingCalls === 1 + ? [] + : [{ name: 'TRACKER', type: 'durable_object_namespace', namespace_id: 'ns-expected' }], + }); + }); + vi.stubGlobal('fetch', fetchMock); + + await expect( + verifyManagedDurableObjectApplicationBindings(env, 'worker-1', manifest, { delaysMs: [0] }), + ).resolves.toBeUndefined(); + expect(bindingCalls).toBe(2); + }); + + it('rejects when bindings never match across all retry attempts', async () => { + const fetchMock = vi.fn(async () => + Response.json({ + success: true, + errors: [], + result: [{ name: 'TRACKER', type: 'durable_object_namespace', namespace_id: 'ns-stale' }], + }), + ); + vi.stubGlobal('fetch', fetchMock); + + await expect( + verifyManagedDurableObjectApplicationBindings(env, 'worker-1', manifest, { + delaysMs: [0, 0, 0], + }), + ).rejects.toThrow('Application custom Durable Object binding could not be verified.'); + expect(fetchMock).toHaveBeenCalledTimes(4); + }); + + it.each(['response body', 'retry delay'])('honors the deadline during a %s', async (phase) => { + vi.useFakeTimers({ toFake: ['Date', 'setTimeout', 'clearTimeout'] }); + const fetchMock = vi.fn(async (_url: string, init: RequestInit) => { + if (phase === 'retry delay') { + return Response.json({ success: true, errors: [], result: [] }); + } + const signal = init.signal!; + return new Response( + new ReadableStream({ + start(controller) { + signal.addEventListener('abort', () => controller.error(signal.reason), { once: true }); + }, + }), + ); + }); + vi.stubGlobal('fetch', fetchMock); + + const rejected = expect( + verifyManagedDurableObjectApplicationBindings(env, 'worker-1', manifest, { + deadline: Date.now() + 100, + }), + ).rejects.toThrow( + 'The deployment reached its time limit while verifying application Durable Object bindings.', + ); + await vi.advanceTimersByTimeAsync(100); + await rejected; + expect(fetchMock).toHaveBeenCalledTimes(1); + }); +});