diff --git a/package.json b/package.json index da11d70..bec2b4d 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,8 @@ "dependencies": { "@oclif/core": "^4", "@salesforce/core": "^8.2.7", - "@salesforce/sf-plugins-core": "^12" + "@salesforce/sf-plugins-core": "^12", + "jszip": "^3.10.1" }, "devDependencies": { "@oclif/plugin-command-snapshot": "^5.2.3", diff --git a/src/base/zipBase.ts b/src/base/zipBase.ts index a2f3745..2d2ce9f 100644 --- a/src/base/zipBase.ts +++ b/src/base/zipBase.ts @@ -16,20 +16,34 @@ import { existsSync } from 'node:fs'; import { SfCommand } from '@salesforce/sf-plugins-core'; import { Messages, SfError } from '@salesforce/core'; -import { DatacodeBinaryExecutor, type DatacodeZipExecutionResult } from '../utils/datacodeBinaryExecutor.js'; -import { checkEnvironment } from '../utils/environmentChecker.js'; -import { type SharedResultProps } from './types.js'; +import { zipWithSfError, type ZipResult as ZipBuilderResult } from '../utils/zipBuilder.js'; export type BaseZipFlags = { 'package-dir': string; network?: string; }; -export type ZipResult = SharedResultProps & { - archivePath?: string; - executionResult?: DatacodeZipExecutionResult; +export type ZipResult = { + success: boolean; + codeType: 'script' | 'function'; + packageDir: string; + archivePath: string; + fileCount: number; + archiveSizeBytes: number; + message: string; }; +function formatBytes(bytes: number): string { + const units = ['B', 'KB', 'MB', 'GB']; + let value = bytes; + let unit = 0; + while (value >= 1024 && unit < units.length - 1) { + value /= 1024; + unit += 1; + } + return `${value.toFixed(unit === 0 ? 0 : 2)} ${units[unit]}`; +} + // eslint-disable-next-line sf-plugin/command-summary, sf-plugin/command-example export abstract class ZipBase extends SfCommand { public static enableJsonFlag = false; @@ -39,7 +53,7 @@ export abstract class ZipBase extends SfCommand { const codeType = this.getCodeType(); const messages = this.getMessages(); const packageDir = flags['package-dir']; - const network = flags.network; + const network = flags.network ?? 'default'; if (!existsSync(packageDir)) { throw new SfError( @@ -50,45 +64,25 @@ export abstract class ZipBase extends SfCommand { } try { - const { pythonInfo, packageInfo, binaryInfo } = await checkEnvironment( - this.spinner, - this.log.bind(this), - messages - ); - this.spinner.start(messages.getMessage('info.executingZip')); - const executionResult = await DatacodeBinaryExecutor.executeBinaryZip(packageDir, network); - + const result: ZipBuilderResult = await zipWithSfError(packageDir, network, this.log.bind(this)); this.spinner.stop(); - if (executionResult.archivePath) { - this.log(messages.getMessage('info.archiveCreated', [executionResult.archivePath])); - } - - if (executionResult.fileCount !== undefined) { - this.log(messages.getMessage('info.filesIncluded', [executionResult.fileCount.toString()])); - } - - if (executionResult.archiveSize) { - this.log(messages.getMessage('info.archiveSize', [executionResult.archiveSize])); - } + this.log(messages.getMessage('info.archiveCreated', [result.archivePath])); + this.log(messages.getMessage('info.filesIncluded', [result.fileCount.toString()])); + this.log(messages.getMessage('info.archiveSize', [formatBytes(result.archiveSizeBytes)])); return { success: true, - pythonVersion: pythonInfo, - packageInfo, - binaryInfo, codeType, packageDir, - archivePath: executionResult.archivePath, - executionResult, + archivePath: result.archivePath, + fileCount: result.fileCount, + archiveSizeBytes: result.archiveSizeBytes, message: messages.getMessage('info.zipCompleted'), }; } catch (error) { this.spinner.stop(); - - // The error will be properly handled by the Salesforce CLI framework - // as an SfError with actions, so we just throw it throw error; } } diff --git a/src/index.ts b/src/index.ts index 45abc7c..5e33385 100644 --- a/src/index.ts +++ b/src/index.ts @@ -19,7 +19,6 @@ export { PipChecker, type PipPackageInfo } from './utils/pipChecker.js'; export { DatacodeBinaryChecker, type DatacodeBinaryInfo } from './utils/datacodeBinaryChecker.js'; export { DatacodeBinaryExecutor, - type DatacodeZipExecutionResult, type DatacodeDeployExecutionResult, type DatacodeRunExecutionResult, } from './utils/datacodeBinaryExecutor.js'; @@ -39,3 +38,11 @@ export { type ScanPermissions, } from './utils/nativeScan.js'; export type { ScanResult } from './base/scanBase.js'; +export { + createZip, + hasNonemptyRequirementsFile, + prepareDependencyArchive, + zip as zipPackage, + zipWithSfError, + type ZipResult as ZipBuilderResult, +} from './utils/zipBuilder.js'; diff --git a/src/utils/datacodeBinaryExecutor.ts b/src/utils/datacodeBinaryExecutor.ts index 154d949..f0bd07b 100644 --- a/src/utils/datacodeBinaryExecutor.ts +++ b/src/utils/datacodeBinaryExecutor.ts @@ -24,14 +24,6 @@ import { spawnAsync, type SpawnError } from './spawnHelper.js'; Messages.importMessagesDirectoryFromMetaUrl(import.meta.url); const messages = Messages.loadMessages('@salesforce/plugin-data-code-extension', 'datacodeBinaryExecutor'); -export type DatacodeZipExecutionResult = { - stdout: string; - stderr: string; - archivePath?: string; - fileCount?: number; - archiveSize?: string; -}; - export type DatacodeDeployExecutionResult = { stdout: string; stderr: string; @@ -48,43 +40,6 @@ export type DatacodeRunExecutionResult = { }; export class DatacodeBinaryExecutor { - /** - * Executes datacustomcode zip with the specified parameters. - * - * @param packageDir The directory containing the initialized package to zip - * @param network Optional network configuration for Jupyter notebooks - * @returns Execution result with stdout, stderr, and archive information - * @throws SfError if execution fails - */ - public static async executeBinaryZip(packageDir: string, network?: string): Promise { - const args = ['zip']; - - if (network) { - args.push('--network', network); - } - - args.push(packageDir); - - try { - const { stdout, stderr } = await spawnAsync('datacustomcode', args, { - timeout: 120_000, - }); - - return { - stdout: stdout.trim(), - stderr: stderr.trim(), - }; - } catch (error) { - const spawnError = error as SpawnError; - const binaryOutput = spawnError.stderr?.trim() ?? (error instanceof Error ? error.message : String(error)); - throw new SfError( - messages.getMessage('error.zipExecutionFailed', [packageDir, binaryOutput]), - 'ZipExecutionFailed', - messages.getMessages('actions.zipExecutionFailed') - ); - } - } - /** * Executes datacustomcode deploy with the specified parameters. * diff --git a/src/utils/zipBuilder.ts b/src/utils/zipBuilder.ts new file mode 100644 index 0000000..82d60ab --- /dev/null +++ b/src/utils/zipBuilder.ts @@ -0,0 +1,404 @@ +/* + * Copyright 2026, Salesforce, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { existsSync, mkdtempSync, rmSync, statSync, readFileSync } from 'node:fs'; +import { mkdir, readdir, readFile, stat, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import * as path from 'node:path'; +import { debuglog } from 'node:util'; +import JSZip from 'jszip'; +import { Messages, SfError } from '@salesforce/core'; +import { findBaseDirectory, getPackageType, type CodeType } from './nativeScan.js'; +import { spawnAsync, type SpawnError } from './spawnHelper.js'; + +const debug = debuglog('datacustomcode'); + +Messages.importMessagesDirectoryFromMetaUrl(import.meta.url); +const messages = Messages.loadMessages('@salesforce/plugin-data-code-extension', 'datacodeBinaryExecutor'); + +export const ZIP_FILE_NAME = 'deployment.zip'; +export const DEPENDENCIES_ARCHIVE_NAME = 'native_dependencies'; +export const DEPENDENCIES_ARCHIVE_FULL_NAME = `${DEPENDENCIES_ARCHIVE_NAME}.tar.gz`; +export const DEPENDENCIES_ARCHIVE_PATH = path.join('payload', 'archives', DEPENDENCIES_ARCHIVE_FULL_NAME); +export const PY_FILES_PATH = path.join('payload', 'py-files'); +export const DOCKER_IMAGE_NAME = 'datacloud-custom-code-dependency-builder'; +const PLATFORM_ENV = { DOCKER_DEFAULT_PLATFORM: 'linux/amd64' } as const; + +export type ZipResult = { + archivePath: string; + fileCount: number; + archiveSizeBytes: number; +}; + +/** + * Returns true when `/requirements.txt` exists and has at least + * one non-comment, non-blank line. + * + * Mirrors `has_nonempty_requirements_file` in `datacustomcode/deploy.py`. The + * Python equivalent passes `dirname(payload_dir)` and looks for + * `requirements.txt` next to the SDK config — this signature accepts the same + * resolved base directory so callers stay consistent. + */ +export function hasNonemptyRequirementsFile(baseDirectory: string): boolean { + const requirementsPath = path.join(baseDirectory, 'requirements.txt'); + try { + if (!existsSync(requirementsPath) || !statSync(requirementsPath).isFile()) { + return false; + } + const contents = readFileSync(requirementsPath, 'utf-8'); + for (const line of contents.split(/\r?\n/)) { + const trimmed = line.trim(); + if (trimmed && !trimmed.startsWith('#')) { + return true; + } + } + } catch (err) { + debug('error reading requirements.txt at %s: %o', requirementsPath, err); + } + return false; +} + +export function dockerBuildCmd(network: string): string[] { + // `docker build [OPTIONS] PATH` — every option (including --network) must come + // before the build-context path ('.'), otherwise Docker rejects the trailing + // flag. The Python original appended it after the path; we correct that here. + const args = ['build', '-t', DOCKER_IMAGE_NAME, '--file', 'Dockerfile.dependencies']; + if (network !== 'default') { + args.push('--network', network); + } + args.push('.'); + return args; +} + +export function dockerRunCmd(network: string, tempDir: string): string[] { + // `docker run [OPTIONS] IMAGE [COMMAND]` — options must precede the image name. + // Placing --network after the image would make Docker treat it as the + // in-container command/args, so we insert it before DOCKER_IMAGE_NAME. + // Docker expects forward slashes in the volume mount path, even on Windows. + const mountPath = tempDir.replace(/\\/g, '/'); + const args = ['run', '--rm', '-v', `${mountPath}:/workspace`]; + if (network !== 'default') { + args.push('--network', network); + } + args.push(DOCKER_IMAGE_NAME); + return args; +} + +/** + * Thin seam over the three docker CLI invocations the dependency builder needs. + * Real callers use `defaultDockerRunner`. Tests can pass a fake to assert the + * file-handling logic without requiring a working docker daemon. + */ +export type DockerRunner = { + imageExists: () => Promise; + build: (args: string[], opts: { env: NodeJS.ProcessEnv; cwd: string }) => Promise; + run: (args: string[], opts: { env: NodeJS.ProcessEnv }) => Promise; +}; + +export const defaultDockerRunner: DockerRunner = { + async imageExists(): Promise { + try { + const { stdout } = await spawnAsync('docker', ['images', '-q', DOCKER_IMAGE_NAME]); + return stdout.trim().length > 0; + } catch { + return false; + } + }, + async build(args, opts): Promise { + debug('docker build: %o', args); + await spawnAsync('docker', args, opts); + }, + async run(args, opts): Promise { + debug('docker run: %o', args); + await spawnAsync('docker', args, opts); + }, +}; + +/** + * Throws an actionable SfError when a file the dependency builder relies on is + * missing, instead of letting a downstream copyFile/spawn surface a raw ENOENT. + */ +function assertBuildFileExists(label: string, filePath: string): void { + if (!existsSync(filePath)) { + throw new SfError( + `Cannot build the dependency archive: required file '${label}' was not found at '${filePath}'.`, + 'DependencyBuildFileMissing', + [ + "Run 'init' to scaffold the package, which generates requirements.txt and build_native_dependencies.sh", + 'Confirm requirements.txt, build_native_dependencies.sh, and Dockerfile.dependencies exist in the package base directory', + 'If you removed these files, restore them from the template before zipping a package with dependencies', + ] + ); + } +} + +async function copyFile(src: string, dest: string): Promise { + const data = new Uint8Array(await readFile(src)); + await writeFile(dest, data); +} + +async function copyTree(src: string, dest: string): Promise { + const entries = await readdir(src, { withFileTypes: true }); + await mkdir(dest, { recursive: true }); + await Promise.all( + entries.map((entry) => { + const srcPath = path.join(src, entry.name); + const destPath = path.join(dest, entry.name); + if (entry.isDirectory()) { + return copyTree(srcPath, destPath); + } + if (entry.isFile()) { + return copyFile(srcPath, destPath); + } + return Promise.resolve(); + }) + ); +} + +/** + * Runs the Docker-based dependency builder. For scripts, copies the resulting + * `native_dependencies.tar.gz` into `/payload/archives/`. For + * functions, copies the generated `py-files/` tree into + * `/payload/py-files/`. + * + * Mirrors `prepare_dependency_archive` in `datacustomcode/deploy.py`. Resolves + * `requirements.txt`, `build_native_dependencies.sh`, and the destination + * directories under `baseDirectory` so the function works regardless of the + * caller's current working directory. + */ +export async function prepareDependencyArchive( + baseDirectory: string, + dockerNetwork: string, + packageType: CodeType, + log: (message: string) => void = (): void => {}, + runner: DockerRunner = defaultDockerRunner +): Promise { + const dockerEnv = { ...process.env, ...PLATFORM_ENV }; + + const requirementsSrc = path.join(baseDirectory, 'requirements.txt'); + const buildScriptSrc = path.join(baseDirectory, 'build_native_dependencies.sh'); + const dockerfileSrc = path.join(baseDirectory, 'Dockerfile.dependencies'); + const archiveDest = path.join(baseDirectory, DEPENDENCIES_ARCHIVE_PATH); + const pyFilesDest = path.join(baseDirectory, PY_FILES_PATH); + + // Fail fast with an actionable error rather than a raw ENOENT from a later + // copyFile/spawn if the scaffolding files the builder depends on are missing. + assertBuildFileExists('requirements.txt', requirementsSrc); + assertBuildFileExists('build_native_dependencies.sh', buildScriptSrc); + + const imageExists = await runner.imageExists(); + if (!imageExists) { + // The image build references Dockerfile.dependencies by relative path, so it + // is only required when we actually have to build the image. + assertBuildFileExists('Dockerfile.dependencies', dockerfileSrc); + log(`Building docker image with docker network: ${dockerNetwork}...`); + // The build must run from the base directory where Dockerfile.dependencies lives. + await runner.build(dockerBuildCmd(dockerNetwork), { env: dockerEnv, cwd: baseDirectory }); + } + + const tempDir = mkdtempSync(path.join(tmpdir(), 'datacustomcode-deps-')); + try { + log(`Building dependencies archive with docker network: ${dockerNetwork}`); + await copyFile(requirementsSrc, path.join(tempDir, 'requirements.txt')); + await copyFile(buildScriptSrc, path.join(tempDir, 'build_native_dependencies.sh')); + + await runner.run(dockerRunCmd(dockerNetwork, tempDir), { env: dockerEnv }); + + if (packageType === 'function') { + const sourcePyFiles = path.join(tempDir, 'py-files'); + if (existsSync(sourcePyFiles)) { + log(`py-files directory found at ${sourcePyFiles}. Copying to payload directory...`); + await mkdir(path.dirname(pyFilesDest), { recursive: true }); + if (existsSync(pyFilesDest)) { + rmSync(pyFilesDest, { recursive: true, force: true }); + } + await copyTree(sourcePyFiles, pyFilesDest); + log(`py-files copied to ${pyFilesDest}`); + } else { + log(`No py-files directory found at ${sourcePyFiles}. Skipping py-files copy.`); + } + } else { + const archivesTempPath = path.join(tempDir, DEPENDENCIES_ARCHIVE_FULL_NAME); + await mkdir(path.dirname(archiveDest), { recursive: true }); + await copyFile(archivesTempPath, archiveDest); + log(`Dependencies archived to ${archiveDest}`); + } + } finally { + // ignore_cleanup_errors equivalent: Docker may leave files the host can't + // delete (e.g., on Windows). Files we needed are already copied out. + try { + rmSync(tempDir, { recursive: true, force: true }); + } catch (err) { + debug('temp dir cleanup error (ignored): %o', err); + } + } +} + +async function collectFiles(directory: string): Promise { + // Match Python `os.walk(path)` (default `followlinks=False`) + `zipfile.write`: + // - Real subdirectories are recursed. + // - Directory symlinks are NOT recursed (Python `is_dir(follow_symlinks=False)` is False). + // - Regular files and file symlinks are both included; their contents are read + // through the symlink so the archive stores a regular-file entry (matching + // the SDK's existing server-side unzip handling). + async function walk(current: string): Promise { + const entries = await readdir(current, { withFileTypes: true }); + const nested = await Promise.all( + entries.map(async (entry) => { + const full = path.join(current, entry.name); + if (entry.name === '.DS_Store') { + return []; + } + if (entry.isDirectory()) { + return walk(full); + } + if (entry.isFile()) { + return [full]; + } + if (entry.isSymbolicLink()) { + // Resolve the symlink target. Skip dangling links and links that + // point at directories (Python's default os.walk would treat those + // as files and the subsequent open() would fail). + try { + const targetStat = await stat(full); + if (targetStat.isFile()) { + return [full]; + } + } catch { + return []; + } + } + return []; + }) + ); + return nested.flat(); + } + return walk(directory); +} + +/** + * Creates `deployment.zip` (DEFLATE-compressed) at the current working + * directory containing every file under `directory` except `.DS_Store`. + * Archive entry names are relative to `directory`, matching the Python + * `os.path.relpath(abs_path, directory)` behavior. + */ +export async function createZip(directory: string): Promise { + const archive = new JSZip(); + const files = await collectFiles(directory); + + const entries = await Promise.all( + files.map(async (absPath) => { + const arcname = path.relative(directory, absPath); + // Use forward slashes inside the zip so the archive is portable across + // platforms (Python's zipfile follows the same convention). + const portableName = arcname.split(path.sep).join('/'); + // stat (not lstat) so symlinked files report the target's permissions. + const [data, entryStat] = await Promise.all([readFile(absPath), stat(absPath)]); + return { + portableName, + data: new Uint8Array(data), + mtime: entryStat.mtime, + unixPermissions: entryStat.mode & 0o777, + }; + }) + ); + for (const entry of entries) { + archive.file(entry.portableName, entry.data, { + date: entry.mtime, + createFolders: false, + unixPermissions: entry.unixPermissions, + }); + } + + // Drop the implicit folder entries JSZip materializes for any path with a "/" + // — the Python zipfile reference adds files only, never directory entries, so + // we strip them here to keep the archives byte-comparable in test diffs. + for (const name of Object.keys(archive.files)) { + if (archive.files[name].dir) { + archive.remove(name); + } + } + + const buffer = await archive.generateAsync({ + type: 'uint8array', + compression: 'DEFLATE', + compressionOptions: { level: 6 }, + platform: process.platform === 'win32' ? 'DOS' : 'UNIX', + }); + + await writeFile(ZIP_FILE_NAME, buffer); + return { + archivePath: ZIP_FILE_NAME, + fileCount: files.length, + archiveSizeBytes: buffer.byteLength, + }; +} + +/** + * High-level zip command: optionally builds the dependency archive, then + * creates `deployment.zip`. Mirrors the Python CLI `zip` command in + * `datacustomcode/cli.py`. + */ +export async function zip( + directory: string, + dockerNetwork: string, + log: (message: string) => void = (): void => {} +): Promise { + if (!existsSync(directory)) { + throw new SfError( + messages.getMessage('error.zipExecutionFailed', [directory, `Package directory not found at '${directory}'`]), + 'PackageDirNotFound' + ); + } + + const baseDirectory = findBaseDirectory(directory); + const packageType = await getPackageType(baseDirectory); + + if (hasNonemptyRequirementsFile(baseDirectory)) { + await prepareDependencyArchive(baseDirectory, dockerNetwork, packageType, log); + } else { + log(`Skipping dependency archive: requirements.txt is missing or empty in ${baseDirectory}`); + } + + debug('zipping directory %s', directory); + const result = await createZip(directory); + debug('created zip at %s (%d files, %d bytes)', result.archivePath, result.fileCount, result.archiveSizeBytes); + return result; +} + +/** + * Convenience wrapper that surfaces docker / zip failures as SfError so the + * Salesforce CLI framework renders them with consistent action hints. + */ +export async function zipWithSfError( + directory: string, + dockerNetwork: string, + log?: (message: string) => void +): Promise { + try { + return await zip(directory, dockerNetwork, log); + } catch (error) { + if (error instanceof SfError) { + throw error; + } + const spawnError = error as SpawnError; + const detail = spawnError.stderr?.trim() ?? (error instanceof Error ? error.message : String(error)); + throw new SfError( + messages.getMessage('error.zipExecutionFailed', [directory, detail]), + 'ZipExecutionFailed', + messages.getMessages('actions.zipExecutionFailed') + ); + } +} diff --git a/test/commands/data-code-extension/zip.test.ts b/test/commands/data-code-extension/zip.test.ts index f52aec0..1e0ee29 100644 --- a/test/commands/data-code-extension/zip.test.ts +++ b/test/commands/data-code-extension/zip.test.ts @@ -35,45 +35,20 @@ describe('data-code-extension zip commands', () => { try { const result = await ScriptZip.run(['--package-dir', './test-package']); - // If Python 3.11+ is installed and package is initialized, check the success result + // The TS port no longer shells out to Python, so success no longer depends + // on Python/pip/binary checks. Just verify the structured result fields. expect(result.success).to.be.true; expect(result.codeType).to.equal('script'); expect(result.packageDir).to.equal('./test-package'); - expect(result.pythonVersion).to.have.property('command'); - expect(result.pythonVersion).to.have.property('version'); - expect(result.pythonVersion).to.have.property('major'); - expect(result.pythonVersion).to.have.property('minor'); - expect(result.pythonVersion).to.have.property('patch'); - - // Check package info if present - if (result.packageInfo) { - expect(result.packageInfo).to.have.property('name'); - expect(result.packageInfo).to.have.property('version'); - expect(result.packageInfo).to.have.property('location'); - expect(result.packageInfo).to.have.property('pipCommand'); - } - // Check binary info if present - if (result.binaryInfo) { - expect(result.binaryInfo).to.have.property('command'); - expect(result.binaryInfo).to.have.property('version'); - // path is optional + if (result.archivePath) { + expect(result.archivePath).to.be.a('string'); } - - // Check execution result if present (when all prerequisites are met) - if (result.executionResult) { - expect(result.executionResult).to.have.property('stdout'); - expect(result.executionResult).to.have.property('stderr'); - // Check optional fields - if (result.executionResult.archivePath) { - expect(result.executionResult.archivePath).to.be.a('string'); - } - if (result.executionResult.fileCount !== undefined) { - expect(result.executionResult.fileCount).to.be.a('number'); - } - if (result.executionResult.archiveSize) { - expect(result.executionResult.archiveSize).to.be.a('string'); - } + if (result.fileCount !== undefined) { + expect(result.fileCount).to.be.a('number'); + } + if (result.archiveSizeBytes !== undefined) { + expect(result.archiveSizeBytes).to.be.a('number'); } expect(result.message).to.be.a('string'); @@ -129,7 +104,6 @@ describe('data-code-extension zip commands', () => { // Should return a structured result expect(result).to.be.an('object'); expect(result).to.have.property('success'); - expect(result).to.have.property('pythonVersion'); expect(result).to.have.property('message'); expect(result).to.have.property('packageDir'); // archivePath may or may not be present depending on whether zip succeeded @@ -146,29 +120,18 @@ describe('data-code-extension zip commands', () => { try { const result = await FunctionZip.run(['--package-dir', './test-function']); - // If Python 3.11+ is installed and package is initialized, check the success result expect(result.success).to.be.true; expect(result.codeType).to.equal('function'); expect(result.packageDir).to.equal('./test-function'); - expect(result.pythonVersion).to.have.property('command'); - expect(result.pythonVersion).to.have.property('version'); - // Check package info if present - if (result.packageInfo) { - expect(result.packageInfo).to.have.property('name'); - expect(result.packageInfo).to.have.property('version'); + if (result.archivePath) { + expect(result.archivePath).to.be.a('string'); } - - // Check binary info if present - if (result.binaryInfo) { - expect(result.binaryInfo).to.have.property('command'); - expect(result.binaryInfo).to.have.property('version'); + if (result.fileCount !== undefined) { + expect(result.fileCount).to.be.a('number'); } - - // Check execution result if present - if (result.executionResult) { - expect(result.executionResult).to.have.property('stdout'); - expect(result.executionResult).to.have.property('stderr'); + if (result.archiveSizeBytes !== undefined) { + expect(result.archiveSizeBytes).to.be.a('number'); } expect(result.message).to.be.a('string'); @@ -219,7 +182,6 @@ describe('data-code-extension zip commands', () => { // Should return a structured result expect(result).to.be.an('object'); expect(result).to.have.property('success'); - expect(result).to.have.property('pythonVersion'); expect(result).to.have.property('message'); expect(result).to.have.property('packageDir'); } catch (error) { diff --git a/test/utils/zipBuilder.test.ts b/test/utils/zipBuilder.test.ts new file mode 100644 index 0000000..22a561e --- /dev/null +++ b/test/utils/zipBuilder.test.ts @@ -0,0 +1,465 @@ +/* + * Copyright 2026, Salesforce, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { + mkdtempSync, + mkdirSync, + writeFileSync, + rmSync, + existsSync, + readFileSync, + symlinkSync, + chmodSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import * as path from 'node:path'; +import { expect } from 'chai'; +import JSZip from 'jszip'; +import { + createZip, + dockerBuildCmd, + dockerRunCmd, + hasNonemptyRequirementsFile, + prepareDependencyArchive, + zip, + ZIP_FILE_NAME, + type DockerRunner, +} from '../../src/utils/zipBuilder.js'; + +const SDK_CONFIG_DIR = '.datacustomcode_proj'; +const SDK_CONFIG_FILE = 'sdk_config.json'; + +function makeTempDir(prefix = 'zipBuilder-'): string { + return mkdtempSync(path.join(tmpdir(), prefix)); +} + +function writeSdkConfig(baseDir: string, config: unknown): void { + const sdkDir = path.join(baseDir, SDK_CONFIG_DIR); + mkdirSync(sdkDir, { recursive: true }); + writeFileSync(path.join(sdkDir, SDK_CONFIG_FILE), JSON.stringify(config, null, 2)); +} + +describe('zipBuilder.hasNonemptyRequirementsFile', () => { + let tempDir: string; + + beforeEach(() => { + tempDir = makeTempDir(); + }); + + afterEach(() => { + rmSync(tempDir, { recursive: true, force: true }); + }); + + it('returns false when requirements.txt is missing', () => { + expect(hasNonemptyRequirementsFile(tempDir)).to.equal(false); + }); + + it('returns false when requirements.txt only has comments and blank lines', () => { + writeFileSync(path.join(tempDir, 'requirements.txt'), '# comment\n\n \n# another\n'); + expect(hasNonemptyRequirementsFile(tempDir)).to.equal(false); + }); + + it('returns true when requirements.txt has at least one non-comment line', () => { + writeFileSync(path.join(tempDir, 'requirements.txt'), '# comment\npandas==2.0.0\n'); + expect(hasNonemptyRequirementsFile(tempDir)).to.equal(true); + }); + + it('treats indented comments as comments', () => { + writeFileSync(path.join(tempDir, 'requirements.txt'), ' # indented comment\n'); + expect(hasNonemptyRequirementsFile(tempDir)).to.equal(false); + }); +}); + +describe('zipBuilder docker command builders', () => { + const IMAGE = 'datacloud-custom-code-dependency-builder'; + + it('builds a docker build command without --network for the default network', () => { + expect(dockerBuildCmd('default')).to.deep.equal(['build', '-t', IMAGE, '--file', 'Dockerfile.dependencies', '.']); + }); + + it('places --network before the build-context path so docker parses it as an option', () => { + const args = dockerBuildCmd('host'); + expect(args).to.deep.equal(['build', '-t', IMAGE, '--file', 'Dockerfile.dependencies', '--network', 'host', '.']); + // `docker build [OPTIONS] PATH`: the context '.' must be the final arg and + // --network must precede it, never trail it. + expect(args.indexOf('--network')).to.be.lessThan(args.indexOf('.')); + expect(args[args.length - 1]).to.equal('.'); + }); + + it('builds a docker run command with the temp dir mounted', () => { + expect(dockerRunCmd('default', '/tmp/work')).to.deep.equal(['run', '--rm', '-v', '/tmp/work:/workspace', IMAGE]); + }); + + it('places --network before the image name so docker parses it as an option, not a command', () => { + const args = dockerRunCmd('host', '/tmp/work'); + expect(args).to.deep.equal(['run', '--rm', '-v', '/tmp/work:/workspace', '--network', 'host', IMAGE]); + // `docker run [OPTIONS] IMAGE [COMMAND]`: anything after the image is treated + // as the in-container command. --network must come before the image name. + expect(args.indexOf('--network')).to.be.lessThan(args.indexOf(IMAGE)); + expect(args[args.length - 1]).to.equal(IMAGE); + }); + + it('normalizes Windows-style backslashes in the mount path', () => { + const out = dockerRunCmd('host', 'C:\\Users\\x\\tmp'); + expect(out).to.include('C:/Users/x/tmp:/workspace'); + expect(out.indexOf('--network')).to.be.lessThan(out.indexOf(IMAGE)); + }); +}); + +describe('zipBuilder.createZip', () => { + let tempDir: string; + let originalCwd: string; + + beforeEach(() => { + tempDir = makeTempDir(); + originalCwd = process.cwd(); + process.chdir(tempDir); + }); + + afterEach(() => { + process.chdir(originalCwd); + rmSync(tempDir, { recursive: true, force: true }); + }); + + it('creates deployment.zip with all non-DS_Store files at relative paths', async () => { + const payload = path.join(tempDir, 'payload'); + mkdirSync(path.join(payload, 'sub'), { recursive: true }); + writeFileSync(path.join(payload, 'a.py'), 'print(1)'); + writeFileSync(path.join(payload, 'sub', 'b.py'), 'print(2)'); + writeFileSync(path.join(payload, '.DS_Store'), 'junk'); + + const result = await createZip('payload'); + + expect(result.archivePath).to.equal(ZIP_FILE_NAME); + expect(result.fileCount).to.equal(2); + expect(result.archiveSizeBytes).to.be.greaterThan(0); + expect(existsSync(ZIP_FILE_NAME)).to.equal(true); + + const buf = new Uint8Array(readFileSync(ZIP_FILE_NAME)); + const unzipped = await JSZip.loadAsync(buf); + const names = Object.keys(unzipped.files).sort(); + expect(names).to.deep.equal(['a.py', 'sub/b.py']); + + const aContent = await unzipped.files['a.py'].async('string'); + expect(aContent).to.equal('print(1)'); + }); + + it('writes an empty archive when the directory has no files', async () => { + const payload = path.join(tempDir, 'empty'); + mkdirSync(payload); + + const result = await createZip('empty'); + expect(result.fileCount).to.equal(0); + expect(existsSync(ZIP_FILE_NAME)).to.equal(true); + }); + + it('omits implicit folder entries so output matches the Python zipfile format', async () => { + const payload = path.join(tempDir, 'tree'); + mkdirSync(path.join(payload, 'inner'), { recursive: true }); + writeFileSync(path.join(payload, 'top.py'), 'top'); + writeFileSync(path.join(payload, 'inner', 'nested.py'), 'nested'); + + await createZip('tree'); + const buf = new Uint8Array(readFileSync(ZIP_FILE_NAME)); + const unzipped = await JSZip.loadAsync(buf); + for (const name of Object.keys(unzipped.files)) { + expect(unzipped.files[name].dir, `entry ${name} should not be a directory`).to.equal(false); + } + }); +}); + +describe('zipBuilder.zip orchestration', () => { + let tempDir: string; + let originalCwd: string; + + beforeEach(() => { + tempDir = makeTempDir(); + originalCwd = process.cwd(); + process.chdir(tempDir); + }); + + afterEach(() => { + process.chdir(originalCwd); + rmSync(tempDir, { recursive: true, force: true }); + }); + + it('skips dependency archive when requirements.txt is empty and produces a deployment.zip', async () => { + const project = path.join(tempDir, 'proj'); + const payload = path.join(project, 'payload'); + mkdirSync(payload, { recursive: true }); + writeSdkConfig(project, { type: 'script' }); + writeFileSync(path.join(project, 'requirements.txt'), '# only comments\n'); + writeFileSync(path.join(payload, 'entrypoint.py'), 'pass'); + + const logs: string[] = []; + const result = await zip(path.join('proj', 'payload'), 'default', (m) => logs.push(m)); + + expect(result.fileCount).to.equal(1); + expect(result.archivePath).to.equal(ZIP_FILE_NAME); + expect(logs.some((m) => m.includes('Skipping dependency archive'))).to.equal(true); + }); + + it('throws a clear error when the package directory does not exist', async () => { + let caught: Error | undefined; + try { + await zip(path.join(tempDir, 'missing'), 'default'); + } catch (err) { + caught = err as Error; + } + expect(caught).to.be.instanceOf(Error); + expect(caught!.message).to.match(/Package directory not found/); + }); +}); + +describe('zipBuilder.createZip symlinks and permissions', () => { + let tempDir: string; + let originalCwd: string; + + beforeEach(() => { + tempDir = mkdtempSync(path.join(tmpdir(), 'zipBuilder-symlink-')); + originalCwd = process.cwd(); + process.chdir(tempDir); + }); + + afterEach(() => { + process.chdir(originalCwd); + rmSync(tempDir, { recursive: true, force: true }); + }); + + it('includes file symlinks (reading their target contents) like Python os.walk', async function () { + if (process.platform === 'win32') { + this.skip(); + return; + } + const payload = path.join(tempDir, 'payload'); + mkdirSync(payload); + const targetPath = path.join(tempDir, 'real_target.py'); + writeFileSync(targetPath, 'hello-from-target'); + symlinkSync(targetPath, path.join(payload, 'link.py')); + + const result = await createZip('payload'); + expect(result.fileCount).to.equal(1); + + const buf = new Uint8Array(readFileSync(ZIP_FILE_NAME)); + const unzipped = await JSZip.loadAsync(buf); + expect(Object.keys(unzipped.files)).to.deep.equal(['link.py']); + const content = await unzipped.files['link.py'].async('string'); + expect(content).to.equal('hello-from-target'); + }); + + it('skips broken symlinks rather than failing the whole zip', async function () { + if (process.platform === 'win32') { + this.skip(); + return; + } + const payload = path.join(tempDir, 'payload'); + mkdirSync(payload); + writeFileSync(path.join(payload, 'real.py'), 'real'); + symlinkSync('/nonexistent/path', path.join(payload, 'broken.py')); + + const result = await createZip('payload'); + expect(result.fileCount).to.equal(1); + const buf = new Uint8Array(readFileSync(ZIP_FILE_NAME)); + const unzipped = await JSZip.loadAsync(buf); + expect(Object.keys(unzipped.files)).to.deep.equal(['real.py']); + }); + + it('preserves unix executable permissions on archive entries', async function () { + if (process.platform === 'win32') { + this.skip(); + return; + } + const payload = path.join(tempDir, 'payload'); + mkdirSync(payload); + const scriptPath = path.join(payload, 'run.sh'); + writeFileSync(scriptPath, '#!/bin/bash\necho hi\n'); + chmodSync(scriptPath, 0o755); + + await createZip('payload'); + const buf = new Uint8Array(readFileSync(ZIP_FILE_NAME)); + const unzipped = await JSZip.loadAsync(buf); + const entry = unzipped.files['run.sh']; + expect(entry).to.exist; + expect(entry.unixPermissions).to.equal(0o755); + }); +}); + +describe('zipBuilder.prepareDependencyArchive', () => { + let baseDir: string; + + beforeEach(() => { + baseDir = mkdtempSync(path.join(tmpdir(), 'zipBuilder-prep-')); + writeFileSync(path.join(baseDir, 'requirements.txt'), 'pandas==2.0.0\n'); + writeFileSync(path.join(baseDir, 'build_native_dependencies.sh'), '#!/bin/bash\necho stub\n'); + writeFileSync(path.join(baseDir, 'Dockerfile.dependencies'), 'FROM python:3.11\n'); + }); + + afterEach(() => { + rmSync(baseDir, { recursive: true, force: true }); + }); + + type Calls = { + imageExists: number; + build: Array<{ args: string[]; cwd: string }>; + run: Array<{ args: string[] }>; + }; + + function makeRunner(opts: { imageExists: boolean; onRun?: (mountPath: string) => void }): { + runner: DockerRunner; + calls: Calls; + } { + const calls: Calls = { imageExists: 0, build: [], run: [] }; + const runner: DockerRunner = { + async imageExists() { + calls.imageExists += 1; + return opts.imageExists; + }, + async build(args, runOpts) { + calls.build.push({ args, cwd: runOpts.cwd }); + }, + async run(args) { + calls.run.push({ args }); + // The dockerRunCmd shape is `['run', '--rm', '-v', ':/workspace', ...]` + const mountArg = args[args.indexOf('-v') + 1]; + const [mountPath] = mountArg.split(':'); + opts.onRun?.(mountPath); + }, + }; + return { runner, calls }; + } + + it('copies the script tarball into /payload/archives for script packages', async () => { + const { runner, calls } = makeRunner({ + imageExists: true, + onRun: (mountPath) => { + // Simulate the docker container producing the archive in the mount. + writeFileSync(path.join(mountPath, 'native_dependencies.tar.gz'), 'tar-bytes'); + }, + }); + + await prepareDependencyArchive(baseDir, 'default', 'script', () => {}, runner); + + const archivePath = path.join(baseDir, 'payload', 'archives', 'native_dependencies.tar.gz'); + expect(existsSync(archivePath)).to.equal(true); + expect(readFileSync(archivePath, 'utf-8')).to.equal('tar-bytes'); + // imageExists=true, so we should NOT have built. + expect(calls.build).to.have.length(0); + expect(calls.run).to.have.length(1); + }); + + it('copies py-files into /payload/py-files for function packages', async () => { + const { runner } = makeRunner({ + imageExists: true, + onRun: (mountPath) => { + const pyFilesSrc = path.join(mountPath, 'py-files'); + mkdirSync(pyFilesSrc); + writeFileSync(path.join(pyFilesSrc, 'a.py'), 'a-content'); + mkdirSync(path.join(pyFilesSrc, 'sub')); + writeFileSync(path.join(pyFilesSrc, 'sub', 'b.py'), 'b-content'); + }, + }); + + await prepareDependencyArchive(baseDir, 'default', 'function', () => {}, runner); + + const dest = path.join(baseDir, 'payload', 'py-files'); + expect(existsSync(path.join(dest, 'a.py'))).to.equal(true); + expect(readFileSync(path.join(dest, 'a.py'), 'utf-8')).to.equal('a-content'); + expect(readFileSync(path.join(dest, 'sub', 'b.py'), 'utf-8')).to.equal('b-content'); + }); + + it('builds the docker image when it is missing, scoped to baseDirectory', async () => { + const { runner, calls } = makeRunner({ + imageExists: false, + onRun: (mountPath) => { + writeFileSync(path.join(mountPath, 'native_dependencies.tar.gz'), 'data'); + }, + }); + + await prepareDependencyArchive(baseDir, 'host', 'script', () => {}, runner); + + expect(calls.build).to.have.length(1); + expect(calls.build[0].cwd).to.equal(baseDir); + expect(calls.build[0].args).to.include('--network'); + expect(calls.build[0].args).to.include('host'); + }); + + it('skips the py-files copy when the docker run produces nothing', async () => { + const logs: string[] = []; + const { runner } = makeRunner({ imageExists: true }); + + await prepareDependencyArchive(baseDir, 'default', 'function', (m) => logs.push(m), runner); + + expect(existsSync(path.join(baseDir, 'payload', 'py-files'))).to.equal(false); + expect(logs.some((m) => m.includes('Skipping py-files copy'))).to.equal(true); + }); + + it('throws an actionable error (not ENOENT) when requirements.txt is missing', async () => { + rmSync(path.join(baseDir, 'requirements.txt')); + const { runner, calls } = makeRunner({ imageExists: true }); + + let caught: Error | undefined; + try { + await prepareDependencyArchive(baseDir, 'default', 'script', () => {}, runner); + } catch (err) { + caught = err as Error; + } + + expect(caught, 'expected prepareDependencyArchive to throw').to.exist; + expect(caught!.name).to.equal('DependencyBuildFileMissing'); + expect(caught!.message).to.match(/requirements\.txt/); + // Should fail before doing any docker work. + expect(calls.run).to.have.length(0); + expect(calls.build).to.have.length(0); + }); + + it('throws an actionable error when build_native_dependencies.sh is missing', async () => { + rmSync(path.join(baseDir, 'build_native_dependencies.sh')); + const { runner } = makeRunner({ imageExists: true }); + + let caught: Error | undefined; + try { + await prepareDependencyArchive(baseDir, 'default', 'script', () => {}, runner); + } catch (err) { + caught = err as Error; + } + + expect(caught!.name).to.equal('DependencyBuildFileMissing'); + expect(caught!.message).to.match(/build_native_dependencies\.sh/); + }); + + it('requires Dockerfile.dependencies only when the image must be built', async () => { + rmSync(path.join(baseDir, 'Dockerfile.dependencies')); + + // imageExists=true: the missing Dockerfile should NOT block the run. + const present = makeRunner({ + imageExists: true, + onRun: (mountPath) => writeFileSync(path.join(mountPath, 'native_dependencies.tar.gz'), 'data'), + }); + await prepareDependencyArchive(baseDir, 'default', 'script', () => {}, present.runner); + expect(existsSync(path.join(baseDir, 'payload', 'archives', 'native_dependencies.tar.gz'))).to.equal(true); + + // imageExists=false: now the missing Dockerfile must raise the actionable error. + const missing = makeRunner({ imageExists: false }); + let caught: Error | undefined; + try { + await prepareDependencyArchive(baseDir, 'default', 'script', () => {}, missing.runner); + } catch (err) { + caught = err as Error; + } + expect(caught!.name).to.equal('DependencyBuildFileMissing'); + expect(caught!.message).to.match(/Dockerfile\.dependencies/); + expect(missing.calls.build).to.have.length(0); + }); +});