diff --git a/.changeset/function-binary-atomic-download.md b/.changeset/function-binary-atomic-download.md new file mode 100644 index 00000000000..0ebf7812990 --- /dev/null +++ b/.changeset/function-binary-atomic-download.md @@ -0,0 +1,5 @@ +--- +'@shopify/app': patch +--- + +Fix `spawn ETXTBSY` errors when several CLI processes download and run the same function binary at once diff --git a/packages/app/src/cli/services/function/binaries.test.ts b/packages/app/src/cli/services/function/binaries.test.ts index 9cbdc96ac8c..4081c68c5f0 100644 --- a/packages/app/src/cli/services/function/binaries.test.ts +++ b/packages/app/src/cli/services/function/binaries.test.ts @@ -11,8 +11,10 @@ import { V2_TRAMPOLINE_VERSION, } from './binaries.js' import {fetch, Response} from '@shopify/cli-kit/node/http' -import {fileExists, removeFile, writeFile} from '@shopify/cli-kit/node/fs' +import {fileExists, readdir, readFile, removeFile, renameFile, writeFile} from '@shopify/cli-kit/node/fs' +import {basename, dirname} from '@shopify/cli-kit/node/path' import {describe, expect, test, vi} from 'vitest' +import {PassThrough} from 'stream' import {gzipSync} from 'zlib' const javy = javyBinary() @@ -29,6 +31,16 @@ vi.mock('@shopify/cli-kit/node/http', async () => { } }) +vi.mock('@shopify/cli-kit/node/fs', async (importActual) => { + const actual: any = await importActual() + return {...actual, renameFile: vi.fn(actual.renameFile)} +}) + +async function temporaryDownloads(bin: {path: string}) { + const files = await readdir(dirname(bin.path)) + return files.filter((file) => file.startsWith(`${basename(bin.path)}.`)) +} + describe('javy', () => { test('properties are set correctly', () => { expect(javy.name).toBe('javy') @@ -202,7 +214,7 @@ describe('javy', () => { // Wait for first download to reach fetch and register in downloadsInProgress await fetchStarted - // Simulate file appearing on disk (e.g., non-atomic moveFile creating the destination + // Simulate file appearing on disk (e.g., another process renaming its download into place // while the download is still tracked as in-progress) await writeFile(javy.path, 'incomplete binary') @@ -363,6 +375,76 @@ describe('functionRunner', () => { expect(fetch).toHaveBeenCalledOnce() await expect(fileExists(functionRunner.path)).resolves.toBeTruthy() }) + + test('writes function-runner next to its final path and renames it into place when complete', async () => { + // Given + await removeFile(functionRunner.path) + const gzippedBinary = gzipSync('function-runner binary') + const responseBody = new PassThrough() + responseBody.write(gzippedBinary.subarray(0, 10)) + vi.mocked(fetch).mockResolvedValue(new Response(responseBody)) + + // When + const download = downloadBinary(functionRunner) + + // Then + // The partial download is in the same directory, so the rename can't fall back to a copy into the final + // path, and the final path doesn't exist while the binary is written. + await vi.waitFor(() => expect(temporaryDownloads(functionRunner)).resolves.toHaveLength(1)) + await expect(fileExists(functionRunner.path)).resolves.toBeFalsy() + + responseBody.end(gzippedBinary.subarray(10)) + await download + await expect(readFile(functionRunner.path)).resolves.toBe('function-runner binary') + await expect(temporaryDownloads(functionRunner)).resolves.toEqual([]) + }) + + test('removes the partial download when the download fails', async () => { + // Given + await removeFile(functionRunner.path) + vi.mocked(fetch).mockImplementation(async () => new Response('not a gzipped binary')) + + // When + const download = downloadBinary(functionRunner) + + // Then + await expect(download).rejects.toThrow() + await expect(fileExists(functionRunner.path)).resolves.toBeFalsy() + await expect(temporaryDownloads(functionRunner)).resolves.toEqual([]) + }) + + test('uses the function-runner that another process put in place when the rename fails', async () => { + // Given + await removeFile(functionRunner.path) + vi.mocked(fetch).mockResolvedValue(new Response(gzipSync('function-runner binary'))) + vi.mocked(renameFile).mockImplementationOnce(async () => { + await writeFile(functionRunner.path, 'function-runner from another process') + throw new Error('EPERM: operation not permitted, rename') + }) + + // When + await downloadBinary(functionRunner) + + // Then + expect(fetch).toHaveBeenCalledOnce() + await expect(readFile(functionRunner.path)).resolves.toBe('function-runner from another process') + await expect(temporaryDownloads(functionRunner)).resolves.toEqual([]) + }) + + test('fails when the rename fails and no function-runner is in place', async () => { + // Given + await removeFile(functionRunner.path) + vi.mocked(fetch).mockImplementation(async () => new Response(gzipSync('function-runner binary'))) + vi.mocked(renameFile).mockRejectedValue(new Error('EACCES: permission denied, rename')) + + // When + const download = downloadBinary(functionRunner) + + // Then + await expect(download).rejects.toThrow('EACCES: permission denied, rename') + await expect(fileExists(functionRunner.path)).resolves.toBeFalsy() + await expect(temporaryDownloads(functionRunner)).resolves.toEqual([]) + }) }) describe('wasm-opt', () => { diff --git a/packages/app/src/cli/services/function/binaries.ts b/packages/app/src/cli/services/function/binaries.ts index a83c4c81197..0e1700f99cf 100644 --- a/packages/app/src/cli/services/function/binaries.ts +++ b/packages/app/src/cli/services/function/binaries.ts @@ -1,5 +1,6 @@ import {joinPath, dirname} from '@shopify/cli-kit/node/path' -import {chmod, createFileWriteStream, fileExists, inTemporaryDirectory, mkdir, moveFile} from '@shopify/cli-kit/node/fs' +import {chmod, createFileWriteStream, fileExists, mkdir, removeFile, renameFile} from '@shopify/cli-kit/node/fs' +import {randomUUID} from '@shopify/cli-kit/node/crypto' import {outputDebug} from '@shopify/cli-kit/node/output' import {performActionWithRetryAfterRecovery} from '@shopify/cli-kit/common/retry' import {fetch} from '@shopify/cli-kit/node/http' @@ -239,14 +240,10 @@ export async function downloadBinary(bin: DownloadableBinary) { return } - // Downloads cannot run concurrently with `exec` since the `moveFile` - // operation is not atomic. It will delete the destination file if it exists - // which will cause `exec` to break if it tries to execute the file before - // the source file has been moved. - // We prevent downloads from happening concurrently with `exec` by enforcing - // that only one download happens for an executable. If we get here, we check - // if a download is in progress, wait for it to finish, and return without - // downloading again. If it's not in progress, then we start the download. + // Only one download happens for an executable in this process. If we get + // here, we check if a download is in progress, wait for it to finish, and + // return without downloading again. If it's not in progress, then we start + // the download. const downloadPromise = downloadsInProgress.get(bin.path) if (downloadPromise) { await downloadPromise @@ -295,22 +292,38 @@ async function performDownload(bin: DownloadableBinary) { throw new Error(`Downloading ${bin.name} failed with empty response body`) } - // Download to a temp location and then move the file only after it's fully processed - // so the `isDownloaded` check above will continue to return false if the file hasn't - // been fully processed. - await inTemporaryDirectory(async (tmpDir) => { - const tmpFile = joinPath(tmpDir, 'binary') + // Download next to the binary and rename the file into place only after it's fully processed. The rename + // is atomic because both paths are on the same file system, so no CLI process can see or execute a + // binary that is still open for writing, which fails with ETXTBSY on Linux. + const tmpFile = `${bin.path}.${randomUUID()}.download` + try { const outputStream = createFileWriteStream(tmpFile) await bin.processResponse(responseStream, outputStream) await chmod(tmpFile, 0o775) - await moveFile(tmpFile, bin.path, {overwrite: true}) - }) + await moveIntoPlace(bin, tmpFile) + } finally { + await removeFile(tmpFile) + } }, async () => {}, 2, ) } +async function moveIntoPlace(bin: DownloadableBinary, tmpFile: string) { + try { + await renameFile(tmpFile, bin.path) + } catch (error) { + // Windows refuses to replace a binary that another CLI process is running. Downloads rename only complete + // binaries into place, so the existing one can be used. + if (await fileExists(bin.path)) { + outputDebug(`Using the ${bin.name} binary that another process downloaded to ${bin.path}`) + return + } + throw error + } +} + async function gunzipResponse(responseStream: PipelineSource, outputStream: fs.WriteStream): Promise { const gunzip = gzip.createGunzip() await stream.pipeline(responseStream, gunzip, outputStream)