diff --git a/.changeset/resolved-directory-symlinks.md b/.changeset/resolved-directory-symlinks.md new file mode 100644 index 0000000000..b2c249902e --- /dev/null +++ b/.changeset/resolved-directory-symlinks.md @@ -0,0 +1,6 @@ +--- +'e2b': patch +'@e2b/python-sdk': patch +--- + +Include directory symlink contents in template COPY uploads and cache hashes when symlink resolution is enabled, while honoring ignore rules. Report cyclic or out-of-context resolved paths as build errors at the user copy callsite. diff --git a/packages/js-sdk/src/template/index.ts b/packages/js-sdk/src/template/index.ts index 7b7d9c43a6..1ad9d18e1f 100644 --- a/packages/js-sdk/src/template/index.ts +++ b/packages/js-sdk/src/template/index.ts @@ -634,6 +634,15 @@ export class TemplateBase return this } + /** + * Copy local files or directories into the template. + * + * With `resolveSymlinks: true`, the build follows file and directory links + * within the file context. Detected directory-link cycles or targets outside + * that context fail the build with a `BuildError` before upload. Exclude the + * link, move its target into the context, or leave resolution disabled to + * copy the link itself (the default). + */ copy( src: PathLike | PathLike[], dest: PathLike, diff --git a/packages/js-sdk/src/template/types.ts b/packages/js-sdk/src/template/types.ts index 8c0c6fd8f9..8316dc1f8b 100644 --- a/packages/js-sdk/src/template/types.ts +++ b/packages/js-sdk/src/template/types.ts @@ -339,6 +339,11 @@ export type CopyItem = { forceUpload?: true user?: string mode?: number + /** + * Follow file and directory links within the file context during the build. + * Detected directory-link cycles or out-of-context targets cause a BuildError. + * Defaults to false, preserving the links themselves. + */ resolveSymlinks?: boolean gzip?: boolean } diff --git a/packages/js-sdk/src/template/utils.ts b/packages/js-sdk/src/template/utils.ts index 7c5fc4e5f4..d92f65b23b 100644 --- a/packages/js-sdk/src/template/utils.ts +++ b/packages/js-sdk/src/template/utils.ts @@ -5,7 +5,7 @@ import path from 'node:path' import url from 'node:url' import { parse, type StackFrame } from 'error-stack-parser-es' import { dynamicImport } from '../utils' -import { TemplateError } from '../errors' +import { BuildError, TemplateError } from '../errors' import { BASE_STEP_NAME, FINALIZE_STEP_NAME } from './consts' import { PatternMatcher } from '@e2b/dockerfile-utils' import type { IgnoreLike, Path } from 'glob' @@ -166,8 +166,9 @@ function normalizeIgnorePattern(pattern: string, contextPath: string): string { */ function createIgnoreMatcher( ignorePatterns: string[], - contextPath: string -): IgnoreLike { + contextPath: string, + resolveSymlinks: boolean = false +): IgnoreLike & { isIgnored: (entry: Path) => boolean } { const absoluteContextPath = path.resolve(contextPath) let matcher: PatternMatcher try { @@ -181,15 +182,18 @@ function createIgnoreMatcher( throw new TemplateError((err as Error).message) } - const ignored = (p: Path) => { + const isIgnored = (p: Path) => { const relativePath = p.relativePosix() return relativePath !== '' && matcher.matches(relativePath) } return { - ignored, + isIgnored, + ignored: (p) => + isIgnored(p) && + (!resolveSymlinks || !matcher.mayMatchUnder(p.relativePosix())), childrenIgnored: (p) => - ignored(p) && !matcher.mayMatchUnder(p.relativePosix()), + isIgnored(p) && !matcher.mayMatchUnder(p.relativePosix()), } } @@ -205,11 +209,21 @@ export async function getAllFilesInPath( src: string, contextPath: string, ignorePatterns: string[], - includeDirectories: boolean = true + includeDirectories: boolean = true, + resolveSymlinks: boolean = false ) { - const { glob } = await dynamicImport('glob') + const { glob, escape } = await dynamicImport('glob') const files = new Map() - const ignore = createIgnoreMatcher(ignorePatterns, contextPath) + const linkCandidates = new Map() + const ignore = createIgnoreMatcher( + ignorePatterns, + contextPath, + resolveSymlinks + ) + const addMatch = (entry: Path) => { + if (!ignore.isIgnored(entry)) files.set(entry.fullpath(), entry) + if (resolveSymlinks) linkCandidates.set(entry.fullpath(), entry) + } const globFiles = await glob(src, { ignore, @@ -246,29 +260,89 @@ export async function getAllFilesInPath( if (isUnderWalkedDir(file.relativePosix())) { continue } + if (resolveSymlinks) linkCandidates.set(file.fullpath(), file) if (file.isDirectory()) { walkedDirs.add(file.relativePosix()) // For directories, add the directory itself and all files inside it - if (includeDirectories) { + if (includeDirectories && !ignore.isIgnored(file)) { files.set(file.fullpath(), file) } - const dirPattern = normalizePath( - // When the matched directory is '.', `file.relative()` can be an empty string. - // In that case, we want to match all files under the current directory instead of - // creating an absolute glob like '/**/*' which would traverse the entire filesystem. - path.join(file.relative() || '.', '**/*') - ) + const dirPattern = resolveSymlinks + ? `${escape(file.relativePosix() || '.')}/**/*` + : normalizePath( + // When the matched directory is '.', `file.relative()` can be an empty string. + // In that case, we want to match all files under the current directory instead of + // creating an absolute glob like '/**/*' which would traverse the entire filesystem. + path.join(file.relative() || '.', '**/*') + ) const dirFiles = await glob(dirPattern, { ignore, withFileTypes: true, dot: true, cwd: contextPath, }) - dirFiles.forEach((f) => files.set(f.fullpath(), f)) + dirFiles.forEach(addMatch) } else { // For files, just add the file - files.set(file.fullpath(), file) + addMatch(file) + } + } + + if (resolveSymlinks) { + const contextRoot = path.resolve(contextPath) + let realContextRoot: string | undefined + const resolveInContext = async (file: Path) => { + realContextRoot ??= await fs.promises.realpath(contextRoot) + const realPath = await fs.promises.realpath(file.fullpath()) + const relativeTarget = path.relative(realContextRoot, realPath) + if ( + relativeTarget === '..' || + relativeTarget.startsWith(`..${path.sep}`) || + path.isAbsolute(relativeTarget) + ) { + throw new BuildError( + `Path '${file.relativePosix()}' resolves outside the template context. Move the target inside the context, exclude the link via .dockerignore, or set \`resolveSymlinks: false\` to copy the link itself.` + ) + } + return realPath } + for (const file of files.values()) linkCandidates.set(file.fullpath(), file) + // Keep logical paths so both ignore rules and archive names use the link's + // location, rather than the resolved target's location. + // Ignored links can still contain explicitly re-included descendants. + for (const file of linkCandidates.values()) { + const entry = await fs.promises + .lstat(file.fullpath()) + .catch(() => undefined) + if (!entry?.isSymbolicLink()) continue + const target = await fs.promises + .stat(file.fullpath()) + .catch(() => undefined) + if (!target?.isDirectory()) continue + + const realPath = await resolveInContext(file) + let parent = path.dirname(file.fullpath()) + while (parent !== contextRoot || files.has(parent)) { + if ((await fs.promises.realpath(parent)) === realPath) { + throw new BuildError( + `Cannot resolve cyclic directory symlink '${file.relativePosix()}'. Remove the link, exclude it via .dockerignore, or set \`resolveSymlinks: false\` to copy the link itself.` + ) + } + const next = path.dirname(parent) + if (next === parent) break + parent = next + } + + const linkedFiles = await glob( + `${escape(file.relativePosix() || '.')}/**/*`, + { ignore, withFileTypes: true, dot: true, cwd: contextPath } + ) + walkedDirs.add(file.relativePosix()) + for (const child of linkedFiles) { + addMatch(child) + } + } + for (const file of files.values()) await resolveInContext(file) } // Sort by full path for a deterministic order — the default sort() would @@ -329,7 +403,16 @@ export async function calculateFilesHash( hash.update(content) - const files = await getAllFilesInPath(src, contextPath, ignorePatterns, true) + const files = await getAllFilesInPath( + src, + contextPath, + ignorePatterns, + true, + resolveSymlinks + ).catch((error: unknown) => { + if (error instanceof BuildError && stackTrace) error.stack = stackTrace + throw error + }) if (files.length === 0) { const error = new Error(`No files found in ${srcPath}`) @@ -533,7 +616,8 @@ export async function spoolTarArchive( fileName, fileContextPath, ignorePatterns, - true + true, + resolveSymlinks ) const filePaths = allFiles.map((file) => file.relativePosix()) diff --git a/packages/js-sdk/tests/template/directorySymlinks.test.ts b/packages/js-sdk/tests/template/directorySymlinks.test.ts new file mode 100644 index 0000000000..83da134740 --- /dev/null +++ b/packages/js-sdk/tests/template/directorySymlinks.test.ts @@ -0,0 +1,366 @@ +import { afterEach, beforeEach, expect, test } from 'vitest' +import { createServer, type Server } from 'node:http' +import { mkdtemp, mkdir, rm, symlink, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import type { AddressInfo } from 'node:net' +import * as tar from 'tar' +import { Template } from '../../src' + +let context: string +let server: Server +let endpoint: string +let uploads: Map[] +let hashes: string[] + +beforeEach(async () => { + context = await mkdtemp(join(tmpdir(), 'e2b-directory-links-')) + uploads = [] + hashes = [] + server = createServer((request, response) => { + const chunks: Buffer[] = [] + request.on('data', (chunk: Buffer) => chunks.push(chunk)) + request.on('end', async () => { + const reply = (status: number, payload: unknown) => { + response.writeHead(status, { 'Content-Type': 'application/json' }) + response.end(JSON.stringify(payload)) + } + if (request.method === 'PUT' && request.url === '/upload') { + const members = new Map() + const parser = tar.list({ + onentry: (entry) => { + members.set(entry.path.replace(/\/$/, ''), entry.type) + }, + }) + await new Promise((resolve, reject) => { + parser.on('end', resolve) + parser.on('error', reject) + parser.end(Buffer.concat(chunks)) + }) + uploads.push(members) + reply(200, {}) + } else if (request.url === '/v3/templates') { + reply(202, { + templateID: 'fixture', + buildID: 'build', + public: false, + names: [], + tags: [], + aliases: [], + }) + } else if (request.url?.startsWith('/templates/fixture/files/')) { + hashes.push(request.url.split('/').at(-1)!) + reply(201, { present: false, url: `${endpoint}/upload` }) + } else if (request.url === '/v2/templates/fixture/builds/build') { + reply(202, {}) + } else { + reply(404, {}) + } + }) + }) + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) + endpoint = `http://127.0.0.1:${(server.address() as AddressInfo).port}` +}) + +afterEach(async () => { + await new Promise((resolve) => server.close(() => resolve())) + await rm(context, { recursive: true, force: true }) +}) + +test.skipIf(process.platform === 'win32').each([true, false])( + 'build copies a directory symlink with resolveSymlinks=%s', + async (resolveSymlinks) => { + await mkdir(join(context, 'real')) + await writeFile(join(context, 'real/app.py'), "print('hello')") + await symlink('real', join(context, 'linked')) + const template = new Template({ fileContextPath: context }) + .fromPythonImage('3') + .copy('linked', '/app', { resolveSymlinks }) + await Template.buildInBackground(template, 'fixture', { + apiKey: 'fixture-placeholder', + apiUrl: endpoint, + }) + expect(uploads.at(-1)).toEqual( + new Map( + resolveSymlinks + ? [ + ['linked', 'Directory'], + ['linked/app.py', 'File'], + ] + : [['linked', 'SymbolicLink']] + ) + ) + } +) + +test.skipIf(process.platform === 'win32').each(['linked', 'real'])( + 'build hashes resolved descendants and honors ignore rules for %s', + async (source) => { + await mkdir(join(context, 'real')) + await mkdir(join(context, 'other')) + await writeFile(join(context, 'other/app.py'), 'first') + await writeFile(join(context, 'other/.env'), 'synthetic-secret') + await symlink('real', join(context, 'linked')) + await symlink('../other', join(context, 'real/nested')) + await symlink('../other', join(context, 'real/again')) + const build = () => + Template.buildInBackground( + new Template({ + fileContextPath: context, + fileIgnorePatterns: [`${source}/nested/.env`, `${source}/again/.env`], + }) + .fromPythonImage('3') + .copy(source, '/app', { resolveSymlinks: true }), + 'fixture', + { apiKey: 'fixture-placeholder', apiUrl: endpoint } + ) + await build() + expect(uploads.at(-1)?.get(`${source}/nested/app.py`)).toBe('File') + expect(uploads.at(-1)?.get(`${source}/again/app.py`)).toBe('File') + expect( + [...uploads.at(-1)!.keys()].some((name) => name.endsWith('.env')) + ).toBe(false) + await writeFile(join(context, 'other/.env'), 'a-longer-synthetic-secret') + await build() + expect(hashes.at(-1)).toBe(hashes.at(-2)) + await writeFile(join(context, 'other/app.py'), 'other') + await build() + expect(hashes.at(-1)).not.toBe(hashes.at(-2)) + } +) + +test.skipIf(process.platform === 'win32')( + 'build reports cyclic directory symlinks', + async () => { + await mkdir(join(context, 'real')) + await symlink('.', join(context, 'real/back')) + const template = new Template({ fileContextPath: context }) + .fromPythonImage('3') + .copy('real', '/app', { resolveSymlinks: true }) + await expect( + Template.buildInBackground(template, 'fixture', { + apiKey: 'fixture-placeholder', + apiUrl: endpoint, + }) + ).rejects.toThrow('cyclic directory symlink') + } +) + +test.skipIf(process.platform === 'win32')( + 'build detects cycles through ignored ancestors', + async () => { + await mkdir(join(context, 'real/ignored'), { recursive: true }) + await symlink('..', join(context, 'real/ignored/back')) + const template = new Template({ + fileContextPath: context, + fileIgnorePatterns: ['**/ignored', '!**/back'], + }) + .fromPythonImage('3') + .copy('real', '/app', { resolveSymlinks: true }) + await expect( + Template.buildInBackground(template, 'fixture', { + apiKey: 'fixture-placeholder', + apiUrl: endpoint, + }) + ).rejects.toThrow('cyclic directory symlink') + } +) + +test.skipIf(process.platform === 'win32')( + 'build reincludes children under an ignored directory link', + async () => { + await mkdir(join(context, 'real')) + await mkdir(join(context, 'target')) + await writeFile(join(context, 'target/keep.txt'), 'keep') + await writeFile(join(context, 'target/skip.txt'), 'skip') + await symlink('../target', join(context, 'real/link')) + const build = () => + Template.buildInBackground( + new Template({ + fileContextPath: context, + fileIgnorePatterns: ['target', 'real/link', '!real/link/keep.txt'], + }) + .fromPythonImage('3') + .copy('.', '/app', { resolveSymlinks: true }), + 'fixture', + { apiKey: 'fixture-placeholder', apiUrl: endpoint } + ) + await build() + expect(uploads.at(-1)?.get('real/link/keep.txt')).toBe('File') + expect(uploads.at(-1)?.has('real/link')).toBe(false) + expect(uploads.at(-1)?.has('real/link/skip.txt')).toBe(false) + await writeFile(join(context, 'target/keep.txt'), 'edit') + await build() + expect(hashes.at(-1)).not.toBe(hashes.at(-2)) + await writeFile(join(context, 'target/skip.txt'), 'ignored edit') + await build() + expect(hashes.at(-1)).toBe(hashes.at(-2)) + } +) + +test.skipIf(process.platform === 'win32')( + 'build resolves matched link names as literal paths', + async () => { + await mkdir(join(context, 'real')) + await mkdir(join(context, 'alias1')) + await writeFile(join(context, 'real/app.py'), 'correct') + await writeFile(join(context, 'alias1/wrong.txt'), 'other directory') + await symlink('real', join(context, 'alias[1]')) + const template = new Template({ fileContextPath: context }) + .fromPythonImage('3') + .copy('*', '/app', { resolveSymlinks: true }) + await Template.buildInBackground(template, 'fixture', { + apiKey: 'fixture-placeholder', + apiUrl: endpoint, + }) + expect(uploads.at(-1)?.get('alias[1]/app.py')).toBe('File') + expect(uploads.at(-1)?.get('alias[1]/wrong.txt')).toBeUndefined() + } +) + +test.skipIf(process.platform === 'win32').each([ + [true, 'directory'], + [false, 'directory'], + [true, 'nested_file'], + [false, 'nested_file'], +] as const)( + 'build keeps resolved link targets within the context: %s %s', + async (resolveSymlinks, targetKind) => { + const outside = await mkdtemp(join(tmpdir(), 'e2b-outside-link-')) + try { + await writeFile(join(outside, 'synthetic.txt'), 'outside fixture') + if (targetKind === 'directory') { + await symlink(outside, join(context, 'linked')) + } else { + await mkdir(join(context, 'real')) + await symlink( + join(outside, 'synthetic.txt'), + join(context, 'real/external.txt') + ) + await symlink('real', join(context, 'linked')) + } + const template = new Template({ fileContextPath: context }) + .fromPythonImage('3') + .copy('linked', '/app', { resolveSymlinks }) + const result = Template.buildInBackground(template, 'fixture', { + apiKey: 'fixture-placeholder', + apiUrl: endpoint, + }) + if (resolveSymlinks) { + await expect(result).rejects.toThrow(/outside.*context/) + expect(uploads).toHaveLength(0) + } else { + await result + expect(uploads.at(-1)).toEqual(new Map([['linked', 'SymbolicLink']])) + } + } finally { + await rm(outside, { recursive: true, force: true }) + } + } +) + +test.skipIf(process.platform === 'win32')( + 'build link error preserves the copy callsite', + async () => { + await mkdir(join(context, 'real')) + await symlink('.', join(context, 'real/back')) + function copyWithCyclicLink() { + return new Template({ fileContextPath: context }) + .fromPythonImage('3') + .copy('real', '/app', { resolveSymlinks: true }) + } + const template = copyWithCyclicLink() + await expect( + Template.buildInBackground(template, 'fixture', { + apiKey: 'fixture-placeholder', + apiUrl: endpoint, + }) + ).rejects.toMatchObject({ + name: 'BuildError', + message: expect.stringContaining('resolveSymlinks: false'), + stack: expect.stringContaining('copyWithCyclicLink'), + }) + } +) + +test.skipIf(process.platform === 'win32').each([true, false])( + 'build selects only the source under a linked context: %s', + async (resolveSymlinks) => { + await writeFile(join(context, 'app.py'), 'selected') + await writeFile(join(context, 'unselected.txt'), 'not selected') + const aliasRoot = await mkdtemp(join(tmpdir(), 'e2b-context-alias-')) + try { + const alias = join(aliasRoot, 'context') + await symlink(context, alias) + const template = new Template({ fileContextPath: alias }) + .fromPythonImage('3') + .copy('app.py', '/app', { resolveSymlinks }) + await Template.buildInBackground(template, 'fixture', { + apiKey: 'fixture-placeholder', + apiUrl: endpoint, + }) + expect(uploads.at(-1)).toEqual(new Map([['app.py', 'File']])) + } finally { + await rm(aliasRoot, { recursive: true, force: true }) + } + } +) + +test.skipIf(process.platform === 'win32')( + 'build ignores an excluded self-referential link', + async () => { + await writeFile(join(context, 'app.py'), 'selected') + await symlink('loop', join(context, 'loop')) + const template = new Template({ + fileContextPath: context, + fileIgnorePatterns: ['loop'], + }) + .fromPythonImage('3') + .copy('.', '/app', { resolveSymlinks: true }) + await Template.buildInBackground(template, 'fixture', { + apiKey: 'fixture-placeholder', + apiUrl: endpoint, + }) + expect(uploads.at(-1)?.get('app.py')).toBe('File') + expect(uploads.at(-1)?.has('loop')).toBe(false) + } +) + +test.skipIf(process.platform === 'win32')( + 'build reincludes linked descendants selected by a recursive pattern', + async () => { + await mkdir(join(context, 'real')) + await writeFile(join(context, 'real/keep.txt'), 'keep') + await symlink('real', join(context, 'linked')) + const template = new Template({ + fileContextPath: context, + fileIgnorePatterns: ['real', 'linked', '!linked/keep.txt'], + }) + .fromPythonImage('3') + .copy('**/*', '/app', { resolveSymlinks: true }) + await Template.buildInBackground(template, 'fixture', { + apiKey: 'fixture-placeholder', + apiUrl: endpoint, + }) + expect(uploads.at(-1)).toEqual(new Map([['linked/keep.txt', 'File']])) + } +) + +test.skipIf(process.platform === 'win32')( + 'build walks a re-included literal directory name', + async () => { + await mkdir(join(context, 'real[1]')) + await writeFile(join(context, 'real[1]/keep.txt'), 'keep') + const template = new Template({ + fileContextPath: context, + fileIgnorePatterns: ['real*', '!**/keep.txt'], + }) + .fromPythonImage('3') + .copy('**/*', '/app', { resolveSymlinks: true }) + await Template.buildInBackground(template, 'fixture', { + apiKey: 'fixture-placeholder', + apiUrl: endpoint, + }) + expect(uploads.at(-1)).toEqual(new Map([['real[1]/keep.txt', 'File']])) + } +) diff --git a/packages/python-sdk/e2b/template/main.py b/packages/python-sdk/e2b/template/main.py index 98d642301f..aea3e28529 100644 --- a/packages/python-sdk/e2b/template/main.py +++ b/packages/python-sdk/e2b/template/main.py @@ -57,11 +57,17 @@ def copy( :param force_upload: Force upload even if files are cached :param user: User and optionally group (user:group) to own the files :param mode: File permissions in octal format (e.g., 0o755) - :param resolve_symlinks: Whether to resolve symlinks + :param resolve_symlinks: Follow file and directory links within the file + context during the build. Defaults to False, preserving the links. :param gzip: Whether to gzip the files before upload (default: True) :return: `TemplateBuilder` class + When resolution is enabled, detected directory-link cycles or targets + outside the file context fail the build with ``BuildException`` before + upload. Exclude the link, move its target into the context, or set + ``resolve_symlinks=False`` to copy the link itself. + Example ```python template.copy('requirements.txt', '/home/user/') diff --git a/packages/python-sdk/e2b/template/utils.py b/packages/python-sdk/e2b/template/utils.py index fd0d536a9e..526bc44e52 100644 --- a/packages/python-sdk/e2b/template/utils.py +++ b/packages/python-sdk/e2b/template/utils.py @@ -11,7 +11,7 @@ from types import TracebackType, FrameType from typing import IO, List, Optional, Union -from e2b.exceptions import TemplateException +from e2b.exceptions import BuildException, TemplateException from e2b.template.consts import BASE_STEP_NAME, FINALIZE_STEP_NAME from e2b_dockerfile_utils import PatternMatcher @@ -170,6 +170,7 @@ def get_all_files_in_path( context_path: str, ignore_patterns: List[str], include_directories: bool = True, + resolve_symlinks: bool = False, ) -> List[str]: """ Get all files for a given path and ignore patterns. @@ -183,11 +184,13 @@ def get_all_files_in_path( :param context_path: Base directory for resolving relative paths :param ignore_patterns: Ignore patterns in `.dockerignore` syntax :param include_directories: Whether to include directories + :param resolve_symlinks: Whether to traverse symbolic links to directories :return: Array of files """ files = set() abs_context_path = os.path.abspath(context_path) + real_context_path = os.path.realpath(abs_context_path) try: matcher = PatternMatcher( [_normalize_ignore_pattern(p, abs_context_path) for p in ignore_patterns], @@ -196,7 +199,37 @@ def get_all_files_in_path( except ValueError as e: raise TemplateException(str(e)) from e - def walk(dir_path: str, dir_relative_path: str) -> None: + def resolve_in_context(file_path: str, relative_path: str) -> str: + real_path = os.path.realpath(file_path) + try: + in_context = ( + os.path.commonpath([real_context_path, real_path]) == real_context_path + ) + except ValueError: + in_context = False + if not in_context: + raise BuildException( + f"Path '{relative_path}' resolves outside the template context. " + "Move the target inside the context, exclude the link via .dockerignore, " + "or set `resolve_symlinks=False` to copy the link itself." + ) + return real_path + + def walk( + dir_path: str, dir_relative_path: str, ancestors: frozenset[str] = frozenset() + ) -> None: + real_path = ( + resolve_in_context(dir_path, dir_relative_path) + if resolve_symlinks + else os.path.realpath(dir_path) + ) + if real_path in ancestors: + raise BuildException( + f"Cannot resolve cyclic directory symlink '{dir_relative_path}'. " + "Remove the link, exclude it via .dockerignore, " + "or set `resolve_symlinks=False` to copy the link itself." + ) + ancestors = ancestors | {real_path} with os.scandir(dir_path) as entries: for entry in entries: relative_path = ( @@ -207,10 +240,10 @@ def walk(dir_path: str, dir_relative_path: str) -> None: ignored = matcher.matches(relative_path) if not ignored: files.add(entry.path) - if entry.is_dir(follow_symlinks=False) and ( + if ( not ignored or matcher.may_match_under(relative_path) - ): - walk(entry.path, relative_path) + ) and entry.is_dir(follow_symlinks=resolve_symlinks): + walk(entry.path, relative_path, ancestors) # Use glob to find all files/directories matching the pattern under context_path files_glob = glob.glob( @@ -245,19 +278,28 @@ def is_under_walked_dir(relative_path: str) -> bool: for file_path, relative_path in matches: if is_under_walked_dir(relative_path): continue - if relative_path != "." and matcher.matches(relative_path): + ignored = relative_path != "." and matcher.matches(relative_path) + if ignored and ( + not resolve_symlinks or not matcher.may_match_under(relative_path) + ): continue - # Symlinks are not followed, the link itself is copied - if os.path.isdir(file_path) and not os.path.islink(file_path): + if os.path.isdir(file_path) and ( + resolve_symlinks or not os.path.islink(file_path) + ): # If it's a directory, add the directory and all entries recursively - if include_directories: + if include_directories and not ignored: files.add(file_path) walk(file_path, relative_path) walked_dirs.add(relative_path) - else: + elif not ignored: files.add(file_path) + if resolve_symlinks: + for file_path in files: + resolve_in_context( + file_path, normalize_path(os.path.relpath(file_path, abs_context_path)) + ) return sorted(list(files)) @@ -295,7 +337,12 @@ def calculate_files_hash( hash_obj.update(content.encode()) - files = get_all_files_in_path(src, context_path, ignore_patterns, True) + try: + files = get_all_files_in_path( + src, context_path, ignore_patterns, True, resolve_symlinks + ) + except BuildException as error: + raise error.with_traceback(stack_trace) if len(files) == 0: raise ValueError(f"No files found in {src_path}").with_traceback(stack_trace) @@ -367,7 +414,7 @@ def tar_file_stream( dereference=resolve_symlinks, ) as tar: files = get_all_files_in_path( - file_name, file_context_path, ignore_patterns, True + file_name, file_context_path, ignore_patterns, True, resolve_symlinks ) for file in files: tar.add( diff --git a/packages/python-sdk/tests/shared/template/test_directory_symlinks.py b/packages/python-sdk/tests/shared/template/test_directory_symlinks.py new file mode 100644 index 0000000000..a31ebe38af --- /dev/null +++ b/packages/python-sdk/tests/shared/template/test_directory_symlinks.py @@ -0,0 +1,470 @@ +import io +import json +import sys +import tarfile +import threading +import traceback +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +import pytest + +from e2b import AsyncTemplate, Template +from e2b.exceptions import BuildException + + +@pytest.fixture +def build_endpoint(): + """Replace only the remote build API and inspect the uploaded archive.""" + uploads = [] + hashes = [] + + class Handler(BaseHTTPRequestHandler): + def reply(self, status, payload): + body = json.dumps(payload).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def do_POST(self): + self.rfile.read(int(self.headers.get("Content-Length", 0))) + if self.path == "/v3/templates": + self.reply( + 202, + { + "templateID": "fixture", + "buildID": "build", + "public": False, + "names": [], + "tags": [], + "aliases": [], + }, + ) + elif self.path == "/v2/templates/fixture/builds/build": + self.reply(202, {}) + else: + self.reply(404, {}) + + def do_GET(self): + if not self.path.startswith("/templates/fixture/files/"): + self.reply(404, {}) + return + hashes.append(self.path.rsplit("/", 1)[-1]) + self.reply( + 201, + { + "present": False, + "url": f"http://127.0.0.1:{server.server_port}/upload", + }, + ) + + def do_PUT(self): + body = self.rfile.read(int(self.headers["Content-Length"])) + with tarfile.open(fileobj=io.BytesIO(body), mode="r:*") as archive: + members = {} + for entry in archive.getmembers(): + contents = None + if entry.isfile(): + file = archive.extractfile(entry) + assert file is not None + with file: + contents = file.read() + members[entry.name.rstrip("/")] = (entry.type, contents) + uploads.append(members) + self.reply(200, {}) + + def log_message(self, *args): + pass + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield f"http://127.0.0.1:{server.server_port}", uploads, hashes + finally: + server.shutdown() + server.server_close() + thread.join() + + +@pytest.mark.skipif(sys.platform == "win32", reason="Symlinks require privileges") +@pytest.mark.asyncio +@pytest.mark.parametrize("template_class", [Template, AsyncTemplate]) +@pytest.mark.parametrize("resolve", [True, False]) +async def test_build_directory_symlink_contents( + tmp_path, build_endpoint, template_class, resolve +): + (tmp_path / "real").mkdir() + (tmp_path / "real" / "app.py").write_text("print('hello')") + (tmp_path / "linked").symlink_to("real", target_is_directory=True) + endpoint, uploads, _ = build_endpoint + template = ( + template_class(file_context_path=tmp_path) + .from_python_image("3") + .copy( + "linked", + "/app", + resolve_symlinks=resolve, + ) + ) + result = template_class.build_in_background( + template, + "fixture", + api_key="fixture-placeholder", + api_url=endpoint, + ) + if template_class is AsyncTemplate: + await result + + if resolve: + assert uploads[-1] == { + "linked": (tarfile.DIRTYPE, None), + "linked/app.py": (tarfile.REGTYPE, b"print('hello')"), + } + else: + assert uploads[-1] == {"linked": (tarfile.SYMTYPE, None)} + + +@pytest.mark.skipif(sys.platform == "win32", reason="Symlinks require privileges") +@pytest.mark.asyncio +@pytest.mark.parametrize("template_class", [Template, AsyncTemplate]) +@pytest.mark.parametrize("source", ["linked", "real"]) +async def test_build_resolved_links_hash_contents_and_apply_ignores( + tmp_path, + build_endpoint, + template_class, + source, +): + (tmp_path / "real").mkdir() + (tmp_path / "other").mkdir() + target = tmp_path / "other" / "app.py" + target.write_text("first") + secret = tmp_path / "other" / ".env" + secret.write_text("synthetic-secret") + (tmp_path / "linked").symlink_to("real", target_is_directory=True) + for name in ["nested", "again"]: + (tmp_path / "real" / name).symlink_to("../other", target_is_directory=True) + endpoint, uploads, hashes = build_endpoint + + async def build(): + template = ( + template_class( + file_context_path=tmp_path, + file_ignore_patterns=[f"{source}/nested/.env", f"{source}/again/.env"], + ) + .from_python_image("3") + .copy(source, "/app", resolve_symlinks=True) + ) + result = template_class.build_in_background( + template, + "fixture", + api_key="fixture-placeholder", + api_url=endpoint, + ) + if template_class is AsyncTemplate: + await result + + await build() + assert uploads[-1][f"{source}/nested/app.py"] == (tarfile.REGTYPE, b"first") + assert uploads[-1][f"{source}/again/app.py"] == (tarfile.REGTYPE, b"first") + assert not any(name.endswith(".env") for name in uploads[-1]) + secret.write_text("a-longer-synthetic-secret") + await build() + assert hashes[-1] == hashes[-2] + target.write_text("other") + await build() + assert hashes[-1] != hashes[-2] + assert uploads[-1][f"{source}/nested/app.py"] == (tarfile.REGTYPE, b"other") + + +@pytest.mark.skipif(sys.platform == "win32", reason="Symlinks require privileges") +@pytest.mark.asyncio +@pytest.mark.parametrize("template_class", [Template, AsyncTemplate]) +async def test_build_reports_cyclic_directory_symlinks( + tmp_path, build_endpoint, template_class +): + (tmp_path / "real").mkdir() + (tmp_path / "real" / "back").symlink_to(".", target_is_directory=True) + endpoint, _, _ = build_endpoint + template = ( + template_class(file_context_path=tmp_path) + .from_python_image("3") + .copy( + "real", + "/app", + resolve_symlinks=True, + ) + ) + with pytest.raises(BuildException, match="cyclic directory symlink"): + result = template_class.build_in_background( + template, + "fixture", + api_key="fixture-placeholder", + api_url=endpoint, + ) + if template_class is AsyncTemplate: + await result + + +@pytest.mark.skipif(sys.platform == "win32", reason="Symlinks require privileges") +@pytest.mark.asyncio +@pytest.mark.parametrize("template_class", [Template, AsyncTemplate]) +async def test_build_detects_cycles_through_ignored_ancestors( + tmp_path, build_endpoint, template_class +): + (tmp_path / "real" / "ignored").mkdir(parents=True) + (tmp_path / "real" / "ignored" / "back").symlink_to("..", target_is_directory=True) + endpoint, _, _ = build_endpoint + template = ( + template_class( + file_context_path=tmp_path, + file_ignore_patterns=["**/ignored", "!**/back"], + ) + .from_python_image("3") + .copy("real", "/app", resolve_symlinks=True) + ) + with pytest.raises(BuildException, match="cyclic directory symlink"): + result = template_class.build_in_background( + template, + "fixture", + api_key="fixture-placeholder", + api_url=endpoint, + ) + if template_class is AsyncTemplate: + await result + + +@pytest.mark.skipif(sys.platform == "win32", reason="Symlinks require privileges") +@pytest.mark.asyncio +@pytest.mark.parametrize("template_class", [Template, AsyncTemplate]) +@pytest.mark.parametrize("case", ["reinclusion", "literal"]) +async def test_build_link_names_and_reincluded_children( + tmp_path, build_endpoint, template_class, case +): + (tmp_path / "real").mkdir() + if case == "reinclusion": + (tmp_path / "target").mkdir() + (tmp_path / "target" / "keep.txt").write_text("keep") + (tmp_path / "target" / "skip.txt").write_text("skip") + (tmp_path / "real" / "link").symlink_to("../target", target_is_directory=True) + source = "." + ignores = ["target", "real/link", "!real/link/keep.txt"] + else: + (tmp_path / "alias1").mkdir() + (tmp_path / "real" / "app.py").write_text("correct") + (tmp_path / "alias1" / "wrong.txt").write_text("other directory") + (tmp_path / "alias[1]").symlink_to("real", target_is_directory=True) + source = "*" + ignores = [] + endpoint, uploads, hashes = build_endpoint + + async def build(): + template = ( + template_class(file_context_path=tmp_path, file_ignore_patterns=ignores) + .from_python_image("3") + .copy(source, "/app", resolve_symlinks=True) + ) + result = template_class.build_in_background( + template, "fixture", api_key="fixture-placeholder", api_url=endpoint + ) + if template_class is AsyncTemplate: + await result + + await build() + if case == "reinclusion": + assert uploads[-1]["real/link/keep.txt"] == (tarfile.REGTYPE, b"keep") + assert "real/link" not in uploads[-1] + assert "real/link/skip.txt" not in uploads[-1] + (tmp_path / "target" / "keep.txt").write_text("edit") + await build() + assert hashes[-1] != hashes[-2] + (tmp_path / "target" / "skip.txt").write_text("ignored edit") + await build() + assert hashes[-1] == hashes[-2] + else: + assert uploads[-1]["alias[1]/app.py"] == (tarfile.REGTYPE, b"correct") + assert "alias[1]/wrong.txt" not in uploads[-1] + + +@pytest.mark.skipif(sys.platform == "win32", reason="Symlinks require privileges") +@pytest.mark.asyncio +@pytest.mark.parametrize("template_class", [Template, AsyncTemplate]) +@pytest.mark.parametrize("resolve", [True, False]) +@pytest.mark.parametrize("target_kind", ["directory", "nested_file"]) +async def test_build_resolved_directory_links_stay_in_context( + tmp_path, + tmp_path_factory, + build_endpoint, + template_class, + resolve, + target_kind, +): + outside = tmp_path_factory.mktemp("outside-link-target") + (outside / "synthetic.txt").write_text("outside fixture") + if target_kind == "directory": + (tmp_path / "linked").symlink_to(outside, target_is_directory=True) + else: + (tmp_path / "real").mkdir() + (tmp_path / "real" / "external.txt").symlink_to(outside / "synthetic.txt") + (tmp_path / "linked").symlink_to("real", target_is_directory=True) + endpoint, uploads, _ = build_endpoint + template = ( + template_class(file_context_path=tmp_path) + .from_python_image("3") + .copy( + "linked", + "/app", + resolve_symlinks=resolve, + ) + ) + + async def build(): + result = template_class.build_in_background( + template, + "fixture", + api_key="fixture-placeholder", + api_url=endpoint, + ) + if template_class is AsyncTemplate: + await result + + if resolve: + with pytest.raises(BuildException, match="outside.*context"): + await build() + assert not uploads + else: + await build() + assert uploads[-1] == {"linked": (tarfile.SYMTYPE, None)} + + +@pytest.mark.skipif(sys.platform == "win32", reason="Symlinks require privileges") +@pytest.mark.asyncio +@pytest.mark.parametrize("template_class", [Template, AsyncTemplate]) +async def test_build_link_error_preserves_copy_callsite( + tmp_path, build_endpoint, template_class +): + (tmp_path / "real").mkdir() + (tmp_path / "real" / "back").symlink_to(".", target_is_directory=True) + endpoint, _, _ = build_endpoint + + def copy_with_cyclic_link(): + return ( + template_class(file_context_path=tmp_path) + .from_python_image("3") + .copy( + "real", + "/app", + resolve_symlinks=True, + ) + ) + + template = copy_with_cyclic_link() + with pytest.raises(BuildException, match="resolve_symlinks=False") as failure: + result = template_class.build_in_background( + template, "fixture", api_key="fixture-placeholder", api_url=endpoint + ) + if template_class is AsyncTemplate: + await result + assert any( + frame.name == "copy_with_cyclic_link" + for frame in traceback.extract_tb(failure.value.__traceback__) + ) + + +@pytest.mark.skipif(sys.platform == "win32", reason="Symlinks require privileges") +@pytest.mark.asyncio +@pytest.mark.parametrize("template_class", [Template, AsyncTemplate]) +async def test_build_ignores_excluded_self_referential_link( + tmp_path, build_endpoint, template_class +): + (tmp_path / "app.py").write_text("selected") + (tmp_path / "loop").symlink_to("loop") + endpoint, uploads, _ = build_endpoint + template = ( + template_class(file_context_path=tmp_path, file_ignore_patterns=["loop"]) + .from_python_image("3") + .copy(".", "/app", resolve_symlinks=True) + ) + result = template_class.build_in_background( + template, "fixture", api_key="fixture-placeholder", api_url=endpoint + ) + if template_class is AsyncTemplate: + await result + assert uploads[-1]["app.py"] == (tarfile.REGTYPE, b"selected") + assert "loop" not in uploads[-1] + + +@pytest.mark.skipif(sys.platform == "win32", reason="Symlinks require privileges") +@pytest.mark.asyncio +@pytest.mark.parametrize("template_class", [Template, AsyncTemplate]) +@pytest.mark.parametrize("resolve", [True, False]) +async def test_build_selects_only_source_under_linked_context( + tmp_path, tmp_path_factory, build_endpoint, template_class, resolve +): + (tmp_path / "app.py").write_text("selected") + (tmp_path / "unselected.txt").write_text("not selected") + alias = tmp_path_factory.mktemp("context-alias") / "context" + alias.symlink_to(tmp_path, target_is_directory=True) + endpoint, uploads, _ = build_endpoint + template = ( + template_class(file_context_path=alias) + .from_python_image("3") + .copy("app.py", "/app", resolve_symlinks=resolve) + ) + result = template_class.build_in_background( + template, "fixture", api_key="fixture-placeholder", api_url=endpoint + ) + if template_class is AsyncTemplate: + await result + assert uploads[-1] == {"app.py": (tarfile.REGTYPE, b"selected")} + + +@pytest.mark.skipif(sys.platform == "win32", reason="Symlinks require privileges") +@pytest.mark.asyncio +@pytest.mark.parametrize("template_class", [Template, AsyncTemplate]) +async def test_build_reincludes_linked_descendants_in_recursive_pattern( + tmp_path, build_endpoint, template_class +): + (tmp_path / "real").mkdir() + (tmp_path / "real" / "keep.txt").write_text("keep") + (tmp_path / "linked").symlink_to("real", target_is_directory=True) + endpoint, uploads, _ = build_endpoint + template = ( + template_class( + file_context_path=tmp_path, + file_ignore_patterns=["real", "linked", "!linked/keep.txt"], + ) + .from_python_image("3") + .copy("**/*", "/app", resolve_symlinks=True) + ) + result = template_class.build_in_background( + template, "fixture", api_key="fixture-placeholder", api_url=endpoint + ) + if template_class is AsyncTemplate: + await result + assert uploads[-1] == {"linked/keep.txt": (tarfile.REGTYPE, b"keep")} + + +@pytest.mark.skipif(sys.platform == "win32", reason="Symlinks require privileges") +@pytest.mark.asyncio +@pytest.mark.parametrize("template_class", [Template, AsyncTemplate]) +async def test_build_walks_reincluded_literal_directory_name( + tmp_path, build_endpoint, template_class +): + (tmp_path / "real[1]").mkdir() + (tmp_path / "real[1]" / "keep.txt").write_text("keep") + endpoint, uploads, _ = build_endpoint + template = ( + template_class( + file_context_path=tmp_path, file_ignore_patterns=["real*", "!**/keep.txt"] + ) + .from_python_image("3") + .copy("**/*", "/app", resolve_symlinks=True) + ) + result = template_class.build_in_background( + template, "fixture", api_key="fixture-placeholder", api_url=endpoint + ) + if template_class is AsyncTemplate: + await result + assert uploads[-1] == {"real[1]/keep.txt": (tarfile.REGTYPE, b"keep")}