diff --git a/packages/metro-resolver/API.md b/packages/metro-resolver/API.md index 7f2a36fd85..12dee233b2 100644 --- a/packages/metro-resolver/API.md +++ b/packages/metro-resolver/API.md @@ -99,4 +99,10 @@ export type ResolveAsset = (dirPath: string, assetName: string, extension: strin export type Result = {readonly type: 'resolved'; readonly resolution: TResolution} | {readonly type: 'failed'; readonly candidates: TCandidates}; +export type VirtualResolution = Readonly<{ + type: 'virtualModule'; + virtualPath: null | undefined | string; + source: string | Buffer; +}>; + ``` diff --git a/packages/metro-resolver/src/index.js b/packages/metro-resolver/src/index.js index f17cbbf4fa..13d239509e 100644 --- a/packages/metro-resolver/src/index.js +++ b/packages/metro-resolver/src/index.js @@ -23,6 +23,7 @@ export type { Resolution, ResolveAsset, Result, + VirtualResolution, } from './types'; import FailedToResolveNameError from './errors/FailedToResolveNameError'; diff --git a/packages/metro-resolver/src/types.js b/packages/metro-resolver/src/types.js index d9c2338ca4..c83b482685 100644 --- a/packages/metro-resolver/src/types.js +++ b/packages/metro-resolver/src/types.js @@ -30,14 +30,39 @@ export type AssetResolution = Readonly<{ export type FileResolution = AssetResolution | SourceFileResolution; /** - * A JS module whose contents are provided out-of-band rather than read from a file - * on disk. + * A JS module whose contents are provided by the resolver rather than read from + * a file on disk. * - * NOTE: Resolving to a virtual module is not yet implemented. This is a reservation. + * Metro derives the module's identity from `virtualPath` and the specifier + * that resolved to it. A change in `source` for the same specifier is a + * modification of the same module, as an edit to a file is. The transform + * cache key is content-addressed, so nothing about a virtual module can go + * stale. + * + * A resolver must produce the same `source` for the same virtual path and + * specifier wherever they are resolved from. Metro throws when two + * resolutions in one traversal disagree. */ export type VirtualResolution = Readonly<{ type: 'virtualModule', - ... + /** + * The path the module notionally lives at. It need not exist on disk. Metro + * appends a suffix derived from `source` to form the module's path, so + * relative and package imports inside the virtual module resolve from this + * path's directory, and path-based transform configuration (Babel + * overrides, `inlineRequires` block lists, and so on) sees a path derived + * from it. A resolver that passes the importing module's own path gets one + * module per importer; a fixed path gets one module per distinct source. + * + * `null` declares a module with no base, as a `data:` URL has: identity is + * the source alone and relative imports inside it are errors. Resolving the + * dependencies of such a module needs a `ResolutionContext` with no + * `originModulePath`, which is a breaking change to the resolver contract, + * so Metro does not accept `null` yet. + */ + virtualPath: ?string, + /** The complete source of the module, as text or UTF-8 bytes. */ + source: string | Buffer, }>; export type FileAndDirCandidates = { diff --git a/packages/metro/src/DeltaBundler/Graph.js b/packages/metro/src/DeltaBundler/Graph.js index 284bb38ffd..38295aeb87 100644 --- a/packages/metro/src/DeltaBundler/Graph.js +++ b/packages/metro/src/DeltaBundler/Graph.js @@ -183,15 +183,39 @@ export class Graph { const allModifiedPaths = new Set(paths); + // Virtual modules already in the graph whose edge now supplies different + // content. Their identity is unchanged, so they are revisited and committed + // like modified files. + const changedVirtualSources = new Map(); + const delta = await this._buildDelta( modifiedPathsInBaseGraph, internalOptions, - // Traverse new or modified paths - absolutePath => - !this.dependencies.has(absolutePath) || - allModifiedPaths.has(absolutePath), + // Traverse new or modified paths, and virtual modules with new content. + (absolutePath, virtualSource) => { + if ( + !this.dependencies.has(absolutePath) || + allModifiedPaths.has(absolutePath) + ) { + return true; + } + const current = this.#virtualSources.get(absolutePath); + if ( + virtualSource?.type === 'buffer' && + current?.type === 'buffer' && + current.sha1 !== virtualSource.sha1 + ) { + changedVirtualSources.set(absolutePath, virtualSource); + return true; + } + return false; + }, ); + for (const path of changedVirtualSources.keys()) { + modifiedPathsInBaseGraph.add(path); + } + // If we have errors we might need to roll back any changes - take // snapshots of all modified modules at the base state. We'll also snapshot // unmodified modules that become unreachable as they are released, so that @@ -280,6 +304,14 @@ export class Graph { throw error; } + // The graph now holds the new content of revisited virtual modules, so + // later traversals compare against, and re-transform with, that content. + for (const [path, virtualSource] of changedVirtualSources) { + if (this.dependencies.has(path)) { + this.#virtualSources.set(path, virtualSource); + } + } + const added = new Map>(); for (const path of delta.added) { added.set(path, nullthrows(this.dependencies.get(path))); @@ -349,15 +381,21 @@ export class Graph { async _buildDelta( pathsToVisit: ReadonlySet, options: InternalOptions, - moduleFilter?: (path: string) => boolean, + moduleFilter?: (path: string, virtualSource: ?VirtualSource) => boolean, ): Promise> { const subGraph = await buildSubgraph(pathsToVisit, this.#virtualSources, { resolve: options.resolve, - shouldTraverse: (dependency: ResolvedDependency) => { + shouldTraverse: ( + dependency: ResolvedDependency, + virtualSource: ?VirtualSource, + ) => { if (options.shallow || isWeakOrLazy(dependency, options)) { return false; } - return moduleFilter == null || moduleFilter(dependency.absolutePath); + return ( + moduleFilter == null || + moduleFilter(dependency.absolutePath, virtualSource) + ); }, transform: async (absolutePath, virtualSource) => { options.onDependencyAdd(); @@ -561,8 +599,8 @@ export class Graph { if (virtualSource) { this.#virtualSources.set(path, virtualSource); } else { - // This dependency may have existed previously as a require.context - - // clean it up. + // This dependency may have existed previously as a require.context or + // virtual module - clean it up. this.#virtualSources.delete(path); } } diff --git a/packages/metro/src/DeltaBundler/__tests__/Graph-test.js b/packages/metro/src/DeltaBundler/__tests__/Graph-test.js index f7f139d522..1ca7470f02 100644 --- a/packages/metro/src/DeltaBundler/__tests__/Graph-test.js +++ b/packages/metro/src/DeltaBundler/__tests__/Graph-test.js @@ -35,6 +35,7 @@ import type {RequireContextParams} from '../../ModuleGraph/worker/collectDependencies'; import type {Result} from '../Graph'; import type { + BundlerResolution, Dependency, MixedOutput, Module, @@ -50,6 +51,7 @@ import type { import {deriveAbsolutePathFromContext} from '../../lib/contextModule'; import CountingSet from '../../lib/CountingSet'; +import {deriveVirtualModulePath} from '../../lib/virtualModule'; import {Graph} from '../Graph'; import {createPathNormalizer} from './test-utils'; import nullthrows from 'nullthrows'; @@ -316,10 +318,25 @@ class TestGraph extends Graph<> { // Get a snapshot of the graph before the traversal. const dependenciesBefore = new Set(this.dependencies.keys()); const modifiedPaths = new Set(files); + // The mocks don't model file contents, but they do model virtual module + // contents: a virtual module whose transform output key changes is + // expected to be reported as modified. + const virtualKeysBefore = new Map( + [...this.dependencies] + .filter(([path]) => path.includes('?virtual=')) + .map(([path, module]) => [path, module.unstable_transformResultKey]), + ); // Mutate the graph and calculate a delta. const delta = await super.traverseDependencies(paths, options); + for (const [path, keyBefore] of virtualKeysBefore) { + const module = this.dependencies.get(path); + if (module != null && module.unstable_transformResultKey !== keyBefore) { + modifiedPaths.add(path); + } + } + // Validate the delta against the current state of the graph. const expectedDelta = computeDelta( dependenciesBefore, @@ -2946,6 +2963,198 @@ describe('only reachable errors are reported', () => { }); }); +describe('virtual modules', () => { + const p = createPathNormalizer(); + // The virtual module is anchored at a path with no file of its own, so the + // transform mock can be keyed on it independently of the real modules. + const virtualPath = p('/virtual.js'); + const sha1 = (source: Buffer) => + require('node:crypto').createHash('sha1').update(source).digest('hex'); + let virtualSource: Buffer; + let localOptions; + + beforeEach(() => { + virtualSource = Buffer.from('v1'); + // Whatever the resolver produces for `virtual:x` is served from + // `virtualSource`, so a test can change the module's content without + // changing its identity. + localOptions = { + ...options, + resolve: ( + from: string, + to: TransformResultDependency, + ): BundlerResolution => + to.name === 'virtual:x' + ? { + type: 'virtualModule', + filePath: deriveVirtualModulePath(virtualPath, to.name), + source: virtualSource, + sha1: sha1(virtualSource), + virtualPath, + } + : options.resolve(from, to), + }; + transformOverrides.set(virtualPath, async (path, context) => { + if (context?.type !== 'buffer') { + throw new Error('expected the virtual module source'); + } + const {source} = context; + return { + dependencies: [], + getSource: () => source, + output: [ + { + data: {code: source.toString(), lineCount: 1, map: []}, + type: 'js/module', + }, + ], + unstable_transformResultKey: path + ' ' + context.sha1, + }; + }); + Actions.addDependency('/foo', virtualPath, {name: 'virtual:x'}); + files.clear(); + }); + + const modulePath = () => deriveVirtualModulePath(virtualPath, 'virtual:x'); + + test('a virtual module is added once and named after its virtual path and specifier', async () => { + expect( + getPaths(await graph.initialTraverseDependencies(localOptions)), + ).toEqual({ + added: new Set(['/bundle', '/foo', '/bar', '/baz', modulePath()]), + modified: new Set(), + deleted: new Set(), + }); + expect(mockTransform).toHaveBeenCalledWith(virtualPath, { + type: 'buffer', + source: virtualSource, + sha1: sha1(virtualSource), + virtualPath, + }); + }); + + test('a content change for the same specifier is a modification of the same module', async () => { + await graph.initialTraverseDependencies(localOptions); + + virtualSource = Buffer.from('v2'); + Actions.modifyFile('/foo'); + + expect( + getPaths(await graph.traverseDependencies([...files], localOptions)), + ).toEqual({ + added: new Set(), + modified: new Set(['/foo', modulePath()]), + deleted: new Set(), + }); + expect( + nullthrows(graph.dependencies.get(modulePath())).getSource().toString(), + ).toBe('v2'); + }); + + test('re-resolving to the same content does not modify the virtual module', async () => { + await graph.initialTraverseDependencies(localOptions); + + Actions.modifyFile('/foo'); + + expect( + getPaths(await graph.traverseDependencies([...files], localOptions)), + ).toEqual({ + added: new Set(), + modified: new Set(['/foo']), + deleted: new Set(), + }); + }); + + test('after a content change, a later edit to the importer does not revisit the virtual module', async () => { + await graph.initialTraverseDependencies(localOptions); + virtualSource = Buffer.from('v2'); + Actions.modifyFile('/foo'); + await graph.traverseDependencies([...files], localOptions); + + mockTransform.mockClear(); + Actions.modifyFile('/foo'); + await graph.traverseDependencies([...files], localOptions); + + expect(mockTransform).not.toHaveBeenCalledWith( + virtualPath, + expect.anything(), + ); + }); + + test('after a content change, revisiting the virtual module directly uses the new content', async () => { + await graph.initialTraverseDependencies(localOptions); + virtualSource = Buffer.from('v2'); + Actions.modifyFile('/foo'); + await graph.traverseDependencies([...files], localOptions); + + // As getModifiedModulesForDeletedPath does for an inverse dependency of a + // deleted file. + files.clear(); + mockTransform.mockClear(); + await graph.traverseDependencies([modulePath()], localOptions); + + expect(mockTransform).toHaveBeenCalledWith(virtualPath, { + type: 'buffer', + source: Buffer.from('v2'), + sha1: sha1(Buffer.from('v2')), + virtualPath, + }); + }); + + test('a content conflict on an optional dependency is still an error', async () => { + Actions.addDependency('/bar', virtualPath, { + name: 'virtual:x', + data: {isOptional: true}, + }); + files.clear(); + const conflictingOptions = { + ...localOptions, + resolve: ( + from: string, + to: TransformResultDependency, + ): BundlerResolution => + to.name === 'virtual:x' + ? { + type: 'virtualModule', + filePath: modulePath(), + source: Buffer.from(from), + sha1: sha1(Buffer.from(from)), + virtualPath, + } + : options.resolve(from, to), + }; + + await expect( + graph.initialTraverseDependencies(conflictingOptions), + ).rejects.toThrow(/was resolved with different contents/); + }); + + test('two importers resolving the same virtual module to different content is an error', async () => { + Actions.addDependency('/bar', virtualPath, {name: 'virtual:x'}); + files.clear(); + const conflictingOptions = { + ...localOptions, + resolve: ( + from: string, + to: TransformResultDependency, + ): BundlerResolution => + to.name === 'virtual:x' + ? { + type: 'virtualModule', + filePath: modulePath(), + source: Buffer.from(from), + sha1: sha1(Buffer.from(from)), + virtualPath, + } + : options.resolve(from, to), + }; + + await expect( + graph.initialTraverseDependencies(conflictingOptions), + ).rejects.toThrow(/was resolved with different contents/); + }); +}); + describe('require.context', () => { // Context modules are derived from and matched against file paths, so use // system paths throughout. diff --git a/packages/metro/src/DeltaBundler/__tests__/buildSubgraph-test.js b/packages/metro/src/DeltaBundler/__tests__/buildSubgraph-test.js index 74c35f191b..e6e0b7a048 100644 --- a/packages/metro/src/DeltaBundler/__tests__/buildSubgraph-test.js +++ b/packages/metro/src/DeltaBundler/__tests__/buildSubgraph-test.js @@ -9,7 +9,12 @@ */ import type {RequireContextParams} from '../../ModuleGraph/worker/collectDependencies'; -import type {ResolvedDependency, TransformResultDependency} from '../types'; +import type { + BundlerResolution, + ResolvedDependency, + TransformResultDependency, + VirtualSource, +} from '../types'; import {buildSubgraph} from '../buildSubgraph'; import {createPathNormalizer} from './test-utils'; @@ -87,7 +92,7 @@ describe('GraphTraversal', () => { }; }), shouldTraverse: jest.fn( - (dependency: ResolvedDependency) => + (dependency: ResolvedDependency, virtualSource: ?VirtualSource) => dependency.data.data.asyncType !== 'weak', ), }; @@ -216,6 +221,57 @@ describe('GraphTraversal', () => { ); }); + test('transforms a virtual module as its virtual path and resolves its dependencies from there', async () => { + const virtualPath = p('/virtual.ts'); + const source = Buffer.from('export default 1;'); + const sha1 = 'b'.repeat(40); + const modulePath = virtualPath + '?virtual=' + 'a'.repeat(40); + transformDeps.set(virtualPath, [makeTransformDep('foo')]); + const resolveFromVirtual = jest.fn( + (from: string, dep: TransformResultDependency): BundlerResolution => ({ + type: 'sourceFile', + filePath: p('/foo'), + }), + ); + const localParams = { + ...params, + resolve: ( + from: string, + dep: TransformResultDependency, + ): BundlerResolution => + from === p('/bundle') + ? { + type: 'virtualModule', + filePath: modulePath, + source, + sha1, + virtualPath, + } + : resolveFromVirtual(from, dep), + }; + + const {moduleData} = await buildSubgraph( + new Set([p('/bundle')]), + new Map(), + localParams, + ); + + // The graph node is the module path with its identity suffix. + expect(moduleData.has(modulePath)).toBe(true); + // The transformer and the resolver see the bare virtual path, so that + // extension checks and relative resolution behave as for a file there. + expect(params.transform).toHaveBeenCalledWith(virtualPath, { + type: 'buffer', + source, + sha1, + virtualPath, + }); + expect(resolveFromVirtual).toHaveBeenCalledWith( + virtualPath, + expect.anything(), + ); + }); + test('returns errors thrown by the transformer', async () => { transformDeps.set(p('/bar'), [makeTransformDep('bad')]); const result = await buildSubgraph( diff --git a/packages/metro/src/DeltaBundler/__tests__/resolver-test.js b/packages/metro/src/DeltaBundler/__tests__/resolver-test.js index 95e0170e80..b5f992cbc4 100644 --- a/packages/metro/src/DeltaBundler/__tests__/resolver-test.js +++ b/packages/metro/src/DeltaBundler/__tests__/resolver-test.js @@ -2571,6 +2571,66 @@ function dep(name: string): TransformResultDependency { }); }); + test('resolves to a virtual module at the given virtual path', async () => { + setMockFileSystem({'index.js': ''}); + resolveRequest.mockReturnValue({ + type: 'virtualModule', + virtualPath: p('/root/index.js'), + source: 'export default 1;', + }); + + resolver = await createResolver({resolver: {resolveRequest}}); + + expect(resolver.resolve(p('/root/index.js'), dep('virtual'))).toEqual({ + type: 'virtualModule', + filePath: expect.stringMatching( + /^.*[\\/]root[\\/]index\.js\?virtual=[0-9a-f]{40}$/, + ), + source: Buffer.from('export default 1;'), + sha1: expect.stringMatching(/^[0-9a-f]{40}$/), + virtualPath: p('/root/index.js'), + }); + }); + + test('accepts the virtual module source as a Buffer, with the same identity as the text', async () => { + setMockFileSystem({'index.js': ''}); + resolveRequest.mockReturnValue({ + type: 'virtualModule', + virtualPath: p('/root/index.js'), + source: 'export default 1;', + }); + resolver = await createResolver({resolver: {resolveRequest}}); + const fromText = resolver.resolve(p('/root/index.js'), dep('virtual')); + + resolveRequest.mockReturnValue({ + type: 'virtualModule', + virtualPath: p('/root/index.js'), + source: Buffer.from('export default 1;'), + }); + resolver = await createResolver({resolver: {resolveRequest}}); + + expect(resolver.resolve(p('/root/index.js'), dep('virtual'))).toEqual( + fromText, + ); + }); + + test('rejects a virtual module with no virtual path, which is not yet supported', async () => { + setMockFileSystem({'index.js': ''}); + resolveRequest.mockReturnValue({ + type: 'virtualModule', + virtualPath: null, + source: 'export default 1;', + }); + + resolver = await createResolver({resolver: {resolveRequest}}); + + expect(() => + resolver.resolve(p('/root/index.js'), dep('virtual')), + ).toThrow( + 'Virtual modules without a virtualPath are not yet supported.', + ); + }); + test('overrides relative paths', async () => { setMockFileSystem({ 'index.js': '', diff --git a/packages/metro/src/DeltaBundler/buildSubgraph.js b/packages/metro/src/DeltaBundler/buildSubgraph.js index dadf39a629..e5d7619dfc 100644 --- a/packages/metro/src/DeltaBundler/buildSubgraph.js +++ b/packages/metro/src/DeltaBundler/buildSubgraph.js @@ -26,13 +26,19 @@ import path from 'node:path'; type Parameters = Readonly<{ resolve: ResolveFn, transform: TransformFn, - shouldTraverse: ResolvedDependency => boolean, + shouldTraverse: (ResolvedDependency, ?VirtualSource) => boolean, }>; +// The content each virtual module was resolved with so far in one traversal, +// so that two edges disagreeing about a module's source is an error rather +// than an order-dependent bundle. +type VirtualContents = Map; + function resolveDependencies( parentPath: string, dependencies: ReadonlyArray, resolve: ResolveFn, + virtualContents: VirtualContents, ): { dependencies: Map, virtualSources: Map, @@ -72,20 +78,36 @@ function resolveDependencies( data: dep, }; } else { + let resolution; try { - maybeResolvedDep = { - absolutePath: resolve(parentPath, dep).filePath, - data: dep, - }; + resolution = resolve(parentPath, dep); } catch (error) { // Ignore unavailable optional dependencies. They are guarded // with a try-catch block and will be handled during runtime. if (dep.data.isOptional !== true) { throw error; } - maybeResolvedDep = { - data: dep, - }; + } + if (resolution == null) { + maybeResolvedDep = {data: dep}; + } else { + if (resolution.type === 'virtualModule') { + const {filePath, source, sha1, virtualPath} = resolution; + const previous = virtualContents.get(filePath); + if (previous != null && previous.sha1 !== sha1) { + throw new Error( + `Virtual module '${filePath}' was resolved with different ` + + `contents from '${previous.parentPath}' and '${parentPath}'. ` + + 'A resolver must produce the same source for the same ' + + 'virtual path and specifier.', + ); + } + virtualContents.set(filePath, {sha1, parentPath}); + // The source travels with the edge that produced it, so it lives + // exactly as long as the module is reachable. + virtualSources.set(key, {type: 'buffer', source, sha1, virtualPath}); + } + maybeResolvedDep = {absolutePath: resolution.filePath, data: dep}; } } @@ -114,6 +136,7 @@ export async function buildSubgraph( const moduleData: Map> = new Map(); const errors: Map = new Map(); const visitedPaths: Set = new Set(); + const virtualContents: VirtualContents = new Map(); async function visit( absolutePath: string, @@ -123,14 +146,21 @@ export async function buildSubgraph( return; } visitedPaths.add(absolutePath); - const transformResult = await transform(absolutePath, virtualSource); + // A virtual module is transformed as, and resolves its dependencies from, + // its virtual path. The identity suffix exists only in the graph. + const sourcePath = + virtualSource?.type === 'buffer' + ? virtualSource.virtualPath + : absolutePath; + const transformResult = await transform(sourcePath, virtualSource); // Get the absolute path of all sub-dependencies (some of them could have been // moved but maintain the same relative path). const resolutionResult = resolveDependencies( - absolutePath, + sourcePath, transformResult.dependencies, resolve, + virtualContents, ); moduleData.set(absolutePath, { @@ -138,19 +168,23 @@ export async function buildSubgraph( ...resolutionResult, }); - await Promise.all( - [...resolutionResult.dependencies.values()] - .filter( - dependency => - isResolvedDependency(dependency) && shouldTraverse(dependency), - ) - .map(dependency => - visit( - dependency.absolutePath, - resolutionResult.virtualSources.get(dependency.data.data.key), - ).catch(error => errors.set(dependency.absolutePath, error)), - ), - ); + const toVisit = []; + for (const dependency of resolutionResult.dependencies.values()) { + if (!isResolvedDependency(dependency)) { + continue; + } + const dependencyVirtualSource = resolutionResult.virtualSources.get( + dependency.data.data.key, + ); + if (shouldTraverse(dependency, dependencyVirtualSource)) { + toVisit.push( + visit(dependency.absolutePath, dependencyVirtualSource).catch(error => + errors.set(dependency.absolutePath, error), + ), + ); + } + } + await Promise.all(toVisit); } await Promise.all( diff --git a/packages/metro/src/DeltaBundler/types.js b/packages/metro/src/DeltaBundler/types.js index 688bfd42ea..afca4369ac 100644 --- a/packages/metro/src/DeltaBundler/types.js +++ b/packages/metro/src/DeltaBundler/types.js @@ -92,13 +92,29 @@ export type Module = Readonly<{ /** * How the transform obtains the source of a module that has no file, as - * supplied by the resolution that produced it. A `require.context` module - * arrives as the parameters the transform expands by enumerating the file map. + * supplied by the resolution that produced it. A virtual module arrives as the + * buffer itself. A `require.context` module arrives as the parameters the + * transform expands by enumerating the file map: a derived module whose inputs + * are discovered at transform time rather than pinned at resolution, which is + * why it has its own invalidation in `Graph.markModifiedContextModules`. */ -export type VirtualSource = Readonly<{ - type: 'requireContext', - requireContext: RequireContext, -}>; +export type VirtualSource = + | Readonly<{type: 'requireContext', requireContext: RequireContext}> + | Readonly<{ + type: 'buffer', + source: Buffer, + /** + * SHA-1 of `source`. A virtual module's identity is independent of its + * content, so this is how the graph notices that an edge now supplies + * different content for an existing module and re-transforms it. + */ + sha1: string, + /** + * The path the module is transformed as and resolves its dependencies + * from. Its graph path adds an identity suffix to this. + */ + virtualPath: string, + }>; export type ModuleData = Readonly<{ dependencies: ReadonlyMap, @@ -162,10 +178,25 @@ export type AllowOptionalDependenciesWithOptions = { export type AllowOptionalDependencies = boolean | AllowOptionalDependenciesWithOptions; -export type BundlerResolution = Readonly<{ - type: 'sourceFile', - filePath: string, -}>; +export type BundlerResolution = + | Readonly<{ + type: 'sourceFile', + filePath: string, + }> + | Readonly<{ + type: 'virtualModule', + /** + * The module's identity in the graph: the virtual path with a suffix + * derived from the specifier. See `lib/virtualModule.js`. + */ + filePath: string, + /** Carried to the transform as the module's input in place of a file read. */ + source: Buffer, + /** SHA-1 of `source`. */ + sha1: string, + /** `filePath` without its identity suffix. */ + virtualPath: string, + }>; export type Options = Readonly<{ resolve: ResolveFn, diff --git a/packages/metro/src/integration_tests/__tests__/virtual-module-test.js b/packages/metro/src/integration_tests/__tests__/virtual-module-test.js new file mode 100644 index 0000000000..6144db1d77 --- /dev/null +++ b/packages/metro/src/integration_tests/__tests__/virtual-module-test.js @@ -0,0 +1,110 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @flow strict-local + * @format + * @oncall react_native + */ + +'use strict'; + +import type {CustomResolver} from 'metro-resolver'; + +const Metro = require('../../..'); +const execBundle = require('../execBundle'); +const MetroConfig = require('metro-config'); +const path = require('node:path'); + +jest.setTimeout(30 * 1000); + +const VIRTUAL_SOURCE = ` +import {increment} from './subdir/counter'; +export default function bump(n) { + return increment() + n; +} +export const ownModuleId = module.id; +`; + +// A custom resolver that answers one specifier with a virtual module anchored +// at the importer, and delegates everything else. +// A virtual path that does not exist on disk, with an extension the transform +// pipeline keys on: the TypeScript here only parses if the transformer sees +// the `.ts` and not the module's identity suffix. +const TYPED_SOURCE = 'export default (n: number): number => n * 2;'; + +const resolveRequest: CustomResolver = (context, moduleName, platform) => { + switch (moduleName) { + case 'virtual:bump': + return { + type: 'virtualModule', + virtualPath: context.originModulePath, + source: VIRTUAL_SOURCE, + }; + case 'virtual:typed': + return { + type: 'virtualModule', + virtualPath: path.join( + path.dirname(context.originModulePath), + 'typed.ts', + ), + source: TYPED_SOURCE, + }; + default: + return context.resolveRequest(context, moduleName, platform); + } +}; + +async function build({dev}: {dev: boolean}) { + const baseConfig = await Metro.loadConfig({ + config: require.resolve('../metro.config.js'), + }); + const config = MetroConfig.mergeConfig(baseConfig, { + resolver: {resolveRequest}, + }); + return Metro.runBuild(config, { + entry: 'virtual-module/index.js', + dev, + minify: !dev, + }); +} + +test('a resolver may answer with a virtual module anchored at the importer', async () => { + const {code} = await build({dev: true}); + expect(execBundle(code)).toEqual({ + // The virtual module's relative import resolved from the importer's + // directory to the shared counter. + fromVirtual: 11, + // Same source, different importer: a separate module instance, sharing the + // counter it imports. + fromSibling: 102, + // The weak id refers to the same module instance as the static require. + fromWeakId: 1003, + weakIdIsOwnModuleId: true, + // A `.ts` virtual path is transformed as TypeScript. + fromTyped: 21, + }); +}); + +test('the virtual module is named after its importer and a hash of its source', async () => { + const {code} = await build({dev: true}); + const names = [ + ...code.matchAll(/"([^"]*virtual-module\/[^"]*\?virtual=[0-9a-f]+)"/g), + ] + .map(match => match[1]) + .sort(); + expect(names).toEqual([ + expect.stringMatching(/^virtual-module\/index\.js\?virtual=[0-9a-f]{40}$/), + expect.stringMatching( + /^virtual-module\/sibling\.js\?virtual=[0-9a-f]{40}$/, + ), + expect.stringMatching(/^virtual-module\/typed\.ts\?virtual=[0-9a-f]{40}$/), + ]); +}); + +test('builds in production', async () => { + const {code} = await build({dev: false}); + expect(execBundle(code)).toMatchObject({fromVirtual: 11, fromSibling: 102}); +}); diff --git a/packages/metro/src/integration_tests/basic_bundle/virtual-module/index.js b/packages/metro/src/integration_tests/basic_bundle/virtual-module/index.js new file mode 100644 index 0000000000..8ae3d1770c --- /dev/null +++ b/packages/metro/src/integration_tests/basic_bundle/virtual-module/index.js @@ -0,0 +1,35 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @format + * @oncall react_native + */ + +declare var require: { + (id: string | number): any, + resolveWeak: (id: string) => string | number, +}; + +const {default: bumpFromSibling} = require('./sibling'); +const {default: double} = require('virtual:typed'); +// `virtual:bump` is resolved by the test's `resolveRequest` to a virtual +// module anchored at this file. Its relative import resolves from here. +const virtual = require('virtual:bump'); + +const weakId = require.resolveWeak('virtual:bump'); + +const dynamicRequire = require; + +module.exports = { + fromVirtual: virtual.default(10), + // Same source, different importer: a separate module instance sharing the + // counter it imports. + fromSibling: bumpFromSibling(100), + // Requiring by the weak id reaches the same instance as the static require. + fromWeakId: dynamicRequire(weakId).default(1000), + weakIdIsOwnModuleId: weakId === virtual.ownModuleId, + fromTyped: double(10.5), +}; diff --git a/packages/metro/src/integration_tests/basic_bundle/virtual-module/sibling.js b/packages/metro/src/integration_tests/basic_bundle/virtual-module/sibling.js new file mode 100644 index 0000000000..441be3faa1 --- /dev/null +++ b/packages/metro/src/integration_tests/basic_bundle/virtual-module/sibling.js @@ -0,0 +1,11 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @format + * @oncall react_native + */ + +module.exports = require('virtual:bump'); diff --git a/packages/metro/src/integration_tests/basic_bundle/virtual-module/subdir/counter.js b/packages/metro/src/integration_tests/basic_bundle/virtual-module/subdir/counter.js new file mode 100644 index 0000000000..f8b8ef0f9b --- /dev/null +++ b/packages/metro/src/integration_tests/basic_bundle/virtual-module/subdir/counter.js @@ -0,0 +1,15 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @format + * @oncall react_native + */ + +let count = 0; + +export function increment(): number { + return ++count; +} diff --git a/packages/metro/src/lib/transformHelpers.js b/packages/metro/src/lib/transformHelpers.js index 82e5fd7db2..59a6f4ad5d 100644 --- a/packages/metro/src/lib/transformHelpers.js +++ b/packages/metro/src/lib/transformHelpers.js @@ -13,9 +13,9 @@ import type Bundler from '../Bundler'; import type {TransformFn, default as DeltaBundler} from '../DeltaBundler'; import type { BundlerResolution, + VirtualSource, TransformInputOptions, TransformResultDependency, - VirtualSource, } from '../DeltaBundler/types'; import type {TransformOptions} from '../DeltaBundler/Worker'; import type {ResolverInputOptions} from '../shared/types'; @@ -155,7 +155,9 @@ export async function getTransformFn( return async (modulePath: string, virtualSource: ?VirtualSource) => { let fileBuffer: ?Buffer; - if (virtualSource?.type === 'requireContext') { + if (virtualSource?.type === 'buffer') { + fileBuffer = virtualSource.source; + } else if (virtualSource?.type === 'requireContext') { const {requireContext} = virtualSource; const graph = await bundler.getDependencyGraph(); diff --git a/packages/metro/src/lib/virtualModule.js b/packages/metro/src/lib/virtualModule.js new file mode 100644 index 0000000000..04bd1cd211 --- /dev/null +++ b/packages/metro/src/lib/virtualModule.js @@ -0,0 +1,42 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @flow strict-local + * @format + * @oncall react_native + */ + +import crypto from 'node:crypto'; + +const VIRTUAL_MODULE_SUFFIX = '?virtual='; + +/** + * The graph identity of a virtual module: its virtual path, followed by a + * suffix derived from a hash of the specifier that resolved to it. + * + * Identity is deliberately independent of the module's source, so that a + * change in what the resolver produces for the same specifier is a + * modification of the same module, as an edit to a file is, rather than a new + * module. The transform cache key is content-addressed separately, and the + * graph re-transforms a virtual module when the source its edge supplies + * changes (see `buildSubgraph`). + * + * The suffix is identity only and is never parsed back. Resolution and + * transformation see the bare virtual path, carried beside the source on the + * dependency edge, so imports inside the virtual module resolve from that + * path's directory and path-based transform configuration, extension checks + * included, behaves as for a file at that path. + */ +export function deriveVirtualModulePath( + virtualPath: string, + specifier: string, +): string { + return ( + virtualPath + + VIRTUAL_MODULE_SUFFIX + + crypto.createHash('sha1').update(specifier).digest('hex') + ); +} diff --git a/packages/metro/src/node-haste/DependencyGraph.js b/packages/metro/src/node-haste/DependencyGraph.js index ee4a63f708..961736da1a 100644 --- a/packages/metro/src/node-haste/DependencyGraph.js +++ b/packages/metro/src/node-haste/DependencyGraph.js @@ -299,9 +299,14 @@ export default class DependencyGraph extends EventEmitter { mapByResolverOptions, resolverOptionsKey, ); - const mapByTarget = getOrCreateMap(mapByOrigin, originKey); - const mapByPlatform = getOrCreateMap(mapByTarget, targetKey); - let resolution: ?BundlerResolution = mapByPlatform.get(platformKey); + // A virtual module may be anchored at the importing module itself, so its + // resolution is memoised per origin file rather than per directory: the + // same specifier in two sibling files may be two modules. Only virtual + // resolutions are stored per file, so the per-directory memo is checked + // first and a hit there needs no second lookup. + let resolution: ?BundlerResolution = + mapByOrigin.get(originKey)?.get(targetKey)?.get(platformKey) ?? + mapByOrigin.get(originModulePath)?.get(targetKey)?.get(platformKey); if (!resolution) { try { @@ -327,7 +332,12 @@ export default class DependencyGraph extends EventEmitter { } } - mapByPlatform.set(platformKey, resolution); + const memoOriginKey = + resolution.type === 'virtualModule' ? originModulePath : originKey; + getOrCreateMap(getOrCreateMap(mapByOrigin, memoOriginKey), targetKey).set( + platformKey, + resolution, + ); return resolution; } diff --git a/packages/metro/src/node-haste/DependencyGraph/ModuleResolution.js b/packages/metro/src/node-haste/DependencyGraph/ModuleResolution.js index d48a00125d..02bda49d42 100644 --- a/packages/metro/src/node-haste/DependencyGraph/ModuleResolution.js +++ b/packages/metro/src/node-haste/DependencyGraph/ModuleResolution.js @@ -25,10 +25,12 @@ import type { } from 'metro-resolver'; import type {PackageForModule, PackageJson} from 'metro-resolver/private/types'; +import {deriveVirtualModulePath} from '../../lib/virtualModule'; import {codeFrameColumns} from '@babel/code-frame'; import invariant from 'invariant'; import * as Resolver from 'metro-resolver'; import createDefaultContext from 'metro-resolver/private/createDefaultContext'; +import crypto from 'node:crypto'; import fs from 'node:fs'; import path from 'node:path'; import util from 'node:util'; @@ -163,7 +165,7 @@ export class ModuleResolver { dependency.name, platform, ); - return this._getFileResolvedModule(result); + return this._getFileResolvedModule(result, dependency.name); } catch (error) { if (error instanceof Resolver.FailedToResolvePathError) { const {candidates} = error; @@ -229,7 +231,10 @@ export class ModuleResolver { /** * TODO: Return Resolution instead of coercing to BundlerResolution here */ - _getFileResolvedModule(resolution: Resolution): BundlerResolution { + _getFileResolvedModule( + resolution: Resolution, + specifier: string, + ): BundlerResolution { switch (resolution.type) { case 'sourceFile': return resolution; @@ -242,8 +247,25 @@ export class ModuleResolver { case 'empty': return this._getEmptyModule(); case 'virtualModule': - // Reserved for future implementation. - throw new Error('Virtual modules are not yet implemented.'); + // A module with no base needs its dependencies resolved with a null + // `originModulePath`, which `ResolutionContext` does not allow yet. + // Lifting this is a breaking change to the resolver contract. + const {virtualPath} = resolution; + invariant( + virtualPath != null, + 'Virtual modules without a virtualPath are not yet supported.', + ); + const source = + typeof resolution.source === 'string' + ? Buffer.from(resolution.source, 'utf8') + : resolution.source; + return { + type: 'virtualModule', + filePath: deriveVirtualModulePath(virtualPath, specifier), + source, + sha1: crypto.createHash('sha1').update(source).digest('hex'), + virtualPath, + }; default: resolution.type as empty; throw new Error('invalid type');