Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/function-binary-atomic-download.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
'@shopify/app': patch
---

Fix `spawn ETXTBSY` errors when several CLI processes download and run the same function binary at once
86 changes: 84 additions & 2 deletions packages/app/src/cli/services/function/binaries.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,10 @@ import {
V2_TRAMPOLINE_VERSION,
} from './binaries.js'
import {fetch, Response} from '@shopify/cli-kit/node/http'
import {fileExists, removeFile, writeFile} from '@shopify/cli-kit/node/fs'
import {fileExists, readdir, readFile, removeFile, renameFile, writeFile} from '@shopify/cli-kit/node/fs'
import {basename, dirname} from '@shopify/cli-kit/node/path'
import {describe, expect, test, vi} from 'vitest'
import {PassThrough} from 'stream'
import {gzipSync} from 'zlib'

const javy = javyBinary()
Expand All @@ -29,6 +31,16 @@ vi.mock('@shopify/cli-kit/node/http', async () => {
}
})

vi.mock('@shopify/cli-kit/node/fs', async (importActual) => {
const actual: any = await importActual()
return {...actual, renameFile: vi.fn(actual.renameFile)}
})

async function temporaryDownloads(bin: {path: string}) {
const files = await readdir(dirname(bin.path))
return files.filter((file) => file.startsWith(`${basename(bin.path)}.`))
}

describe('javy', () => {
test('properties are set correctly', () => {
expect(javy.name).toBe('javy')
Expand Down Expand Up @@ -202,7 +214,7 @@ describe('javy', () => {
// Wait for first download to reach fetch and register in downloadsInProgress
await fetchStarted

// Simulate file appearing on disk (e.g., non-atomic moveFile creating the destination
// Simulate file appearing on disk (e.g., another process renaming its download into place
// while the download is still tracked as in-progress)
await writeFile(javy.path, 'incomplete binary')

Expand Down Expand Up @@ -363,6 +375,76 @@ describe('functionRunner', () => {
expect(fetch).toHaveBeenCalledOnce()
await expect(fileExists(functionRunner.path)).resolves.toBeTruthy()
})

test('writes function-runner next to its final path and renames it into place when complete', async () => {
// Given
await removeFile(functionRunner.path)
const gzippedBinary = gzipSync('function-runner binary')
const responseBody = new PassThrough()
responseBody.write(gzippedBinary.subarray(0, 10))
vi.mocked(fetch).mockResolvedValue(new Response(responseBody))

// When
const download = downloadBinary(functionRunner)

// Then
// The partial download is in the same directory, so the rename can't fall back to a copy into the final
// path, and the final path doesn't exist while the binary is written.
await vi.waitFor(() => expect(temporaryDownloads(functionRunner)).resolves.toHaveLength(1))
await expect(fileExists(functionRunner.path)).resolves.toBeFalsy()

responseBody.end(gzippedBinary.subarray(10))
await download
await expect(readFile(functionRunner.path)).resolves.toBe('function-runner binary')
await expect(temporaryDownloads(functionRunner)).resolves.toEqual([])
})

test('removes the partial download when the download fails', async () => {
// Given
await removeFile(functionRunner.path)
vi.mocked(fetch).mockImplementation(async () => new Response('not a gzipped binary'))

// When
const download = downloadBinary(functionRunner)

// Then
await expect(download).rejects.toThrow()
await expect(fileExists(functionRunner.path)).resolves.toBeFalsy()
await expect(temporaryDownloads(functionRunner)).resolves.toEqual([])
})

test('uses the function-runner that another process put in place when the rename fails', async () => {
// Given
await removeFile(functionRunner.path)
vi.mocked(fetch).mockResolvedValue(new Response(gzipSync('function-runner binary')))
vi.mocked(renameFile).mockImplementationOnce(async () => {
await writeFile(functionRunner.path, 'function-runner from another process')
throw new Error('EPERM: operation not permitted, rename')
})

// When
await downloadBinary(functionRunner)

// Then
expect(fetch).toHaveBeenCalledOnce()
await expect(readFile(functionRunner.path)).resolves.toBe('function-runner from another process')
await expect(temporaryDownloads(functionRunner)).resolves.toEqual([])
})

test('fails when the rename fails and no function-runner is in place', async () => {
// Given
await removeFile(functionRunner.path)
vi.mocked(fetch).mockImplementation(async () => new Response(gzipSync('function-runner binary')))
vi.mocked(renameFile).mockRejectedValue(new Error('EACCES: permission denied, rename'))

// When
const download = downloadBinary(functionRunner)

// Then
await expect(download).rejects.toThrow('EACCES: permission denied, rename')
await expect(fileExists(functionRunner.path)).resolves.toBeFalsy()
await expect(temporaryDownloads(functionRunner)).resolves.toEqual([])
})
})

describe('wasm-opt', () => {
Expand Down
45 changes: 29 additions & 16 deletions packages/app/src/cli/services/function/binaries.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import {joinPath, dirname} from '@shopify/cli-kit/node/path'
import {chmod, createFileWriteStream, fileExists, inTemporaryDirectory, mkdir, moveFile} from '@shopify/cli-kit/node/fs'
import {chmod, createFileWriteStream, fileExists, mkdir, removeFile, renameFile} from '@shopify/cli-kit/node/fs'
import {randomUUID} from '@shopify/cli-kit/node/crypto'
import {outputDebug} from '@shopify/cli-kit/node/output'
import {performActionWithRetryAfterRecovery} from '@shopify/cli-kit/common/retry'
import {fetch} from '@shopify/cli-kit/node/http'
Expand Down Expand Up @@ -239,14 +240,10 @@ export async function downloadBinary(bin: DownloadableBinary) {
return
}

// Downloads cannot run concurrently with `exec` since the `moveFile`
// operation is not atomic. It will delete the destination file if it exists
// which will cause `exec` to break if it tries to execute the file before
// the source file has been moved.
// We prevent downloads from happening concurrently with `exec` by enforcing
// that only one download happens for an executable. If we get here, we check
// if a download is in progress, wait for it to finish, and return without
// downloading again. If it's not in progress, then we start the download.
// Only one download happens for an executable in this process. If we get
// here, we check if a download is in progress, wait for it to finish, and
// return without downloading again. If it's not in progress, then we start
// the download.
const downloadPromise = downloadsInProgress.get(bin.path)
if (downloadPromise) {
await downloadPromise
Expand Down Expand Up @@ -295,22 +292,38 @@ async function performDownload(bin: DownloadableBinary) {
throw new Error(`Downloading ${bin.name} failed with empty response body`)
}

// Download to a temp location and then move the file only after it's fully processed
// so the `isDownloaded` check above will continue to return false if the file hasn't
// been fully processed.
await inTemporaryDirectory(async (tmpDir) => {
const tmpFile = joinPath(tmpDir, 'binary')
// Download next to the binary and rename the file into place only after it's fully processed. The rename
// is atomic because both paths are on the same file system, so no CLI process can see or execute a
// binary that is still open for writing, which fails with ETXTBSY on Linux.
const tmpFile = `${bin.path}.${randomUUID()}.download`
try {
const outputStream = createFileWriteStream(tmpFile)
await bin.processResponse(responseStream, outputStream)
await chmod(tmpFile, 0o775)
await moveFile(tmpFile, bin.path, {overwrite: true})
})
await moveIntoPlace(bin, tmpFile)
} finally {
await removeFile(tmpFile)
}
},
async () => {},
2,
)
}

async function moveIntoPlace(bin: DownloadableBinary, tmpFile: string) {
try {
await renameFile(tmpFile, bin.path)
} catch (error) {
// Windows refuses to replace a binary that another CLI process is running. Downloads rename only complete
// binaries into place, so the existing one can be used.
if (await fileExists(bin.path)) {
outputDebug(`Using the ${bin.name} binary that another process downloaded to ${bin.path}`)
return
}
throw error
}
}

async function gunzipResponse(responseStream: PipelineSource<unknown>, outputStream: fs.WriteStream): Promise<void> {
const gunzip = gzip.createGunzip()
await stream.pipeline(responseStream, gunzip, outputStream)
Expand Down
Loading