From 560519d587fb89c17930e7b1ff1ba6f79e22e234 Mon Sep 17 00:00:00 2001 From: Jeppe Fredsgaard Blaabjerg Date: Sun, 1 Nov 2026 08:51:26 +0100 Subject: [PATCH 1/2] feat(manifest): list the sidecar classpath in JVM load order projects[].classpath now gives each project's classpath in the order the JVM searches it, with one version per module, so the analysis can keep the first class per name instead of every copy. - Maven: the extension emits each module's classpath in the order Maven builds it from the resolved graph, test scope last. - Gradle: in --with-files runs, a configuration's nodes follow its artifact order, read after verification and only over fetched artifacts so nothing else is downloaded. - sbt: nodes already follow dependencyClasspath. - The assembler ranks roots runtime, compile, other variants, test, tooling, and keeps the first version of each module. --- .gitignore | 5 + src/commands/manifest/scripts/assemble.mts | 153 ++++++++++++++++-- .../manifest/scripts/assemble.test.mts | 148 ++++++++++++++++- src/commands/manifest/scripts/facts.mts | 3 +- .../socket/SocketFactsRecordsEngine.java | 28 +++- src/commands/manifest/scripts/records.mts | 19 +++ src/commands/manifest/scripts/sidecar.mts | 5 +- .../manifest/scripts/socket-facts.init.gradle | 106 +++++++++++- .../test/gradle-compat/order/build.gradle | 24 +++ .../test/gradle-compat/order/settings.gradle | 1 + .../test/gradle-compat/smoke-test-order.sh | 37 +++++ .../test/maven-compat/assert-classpath.py | 35 ++++ .../test/maven-compat/project/app2/pom.xml | 26 +++ .../project/app2/src/main/java/demo/App2.java | 4 + .../scripts/test/maven-compat/project/pom.xml | 1 + .../maven-compat/smoke-test-workspaces.sh | 4 +- .../scripts/test/maven-compat/smoke-test.sh | 13 +- .../manifest/scripts/test/run-compat.sh | 1 + 18 files changed, 568 insertions(+), 45 deletions(-) create mode 100644 src/commands/manifest/scripts/test/gradle-compat/order/build.gradle create mode 100644 src/commands/manifest/scripts/test/gradle-compat/order/settings.gradle create mode 100755 src/commands/manifest/scripts/test/gradle-compat/smoke-test-order.sh create mode 100644 src/commands/manifest/scripts/test/maven-compat/assert-classpath.py create mode 100644 src/commands/manifest/scripts/test/maven-compat/project/app2/pom.xml create mode 100644 src/commands/manifest/scripts/test/maven-compat/project/app2/src/main/java/demo/App2.java diff --git a/.gitignore b/.gitignore index 3dd4c3ebf5..3c875633e3 100644 --- a/.gitignore +++ b/.gitignore @@ -40,6 +40,11 @@ test/fixtures/commands/fix/e2e-test-py-temp-* /src/commands/manifest/scripts/test/gradle-compat/project/localrepo/ /src/commands/manifest/scripts/test/gradle-compat/project/records.tsv /src/commands/manifest/scripts/test/gradle-compat/project/workspaces-records.tsv +/src/commands/manifest/scripts/test/gradle-compat/order/.gradle/ +/src/commands/manifest/scripts/test/gradle-compat/order/build/ +/src/commands/manifest/scripts/test/gradle-compat/order/classpath.txt +/src/commands/manifest/scripts/test/gradle-compat/order/localrepo/ +/src/commands/manifest/scripts/test/gradle-compat/order/records.tsv /src/commands/manifest/scripts/test/maven-compat/project/localrepo/ /src/commands/manifest/scripts/test/maven-compat/project/records.tsv /src/commands/manifest/scripts/test/maven-compat/project/target/ diff --git a/src/commands/manifest/scripts/assemble.mts b/src/commands/manifest/scripts/assemble.mts index 81a7d1a19d..e642b411b2 100644 --- a/src/commands/manifest/scripts/assemble.mts +++ b/src/commands/manifest/scripts/assemble.mts @@ -10,7 +10,12 @@ import { } from './facts.mts' import { moduleKey } from './resolution-report.mts' -import type { ParsedRecords, RawCoord, RawProject } from './records.mts' +import type { + ParsedRecords, + RawCoord, + RawProject, + RawRoot, +} from './records.mts' import type { ResolutionReport } from './resolution-report.mts' const PURL_TYPE_MAVEN = 'maven' @@ -44,7 +49,9 @@ type SubprojectGraph = { type PerRoot = { projectKey: string + config: string prod: boolean + // In the root's classpath order. nodes: Map } @@ -93,6 +100,7 @@ export function assembleFacts( [...parsed.projects.values()], [...parsed.excludedProjects.values()], graphs, + buildClasspaths(parsed, perRoot), fileExists, ), } @@ -139,14 +147,19 @@ function buildPerRoot(parsed: ParsedRecords): Map { parent.children.push(childId) } } - out.set(rootId, { projectKey: r.projectKey, prod: r.prod, nodes }) + out.set(rootId, { + projectKey: r.projectKey, + config: r.config, + prod: r.prod, + nodes, + }) } return out } // Components are merged by id across every resolution root; which ids belong // to which subproject is kept separately (classpathByProject) for -// reachability, which needs each subproject's exact classpath. +// reachability. function mergeById(perRoot: Map): Map { const finalNodes = new Map() for (const { nodes, prod } of perRoot.values()) { @@ -395,17 +408,121 @@ function treeRoots( return roots } -function byProject( - projects: RawProject[], - graphs: Map, - ids: (graph: SubprojectGraph) => Iterable, -): Map { - return new Map( - projects.map(p => { - const g = graphs.get(p.projectKey) - return [p.projectKey, g ? [...ids(g)].sort() : []] - }), +// A test configuration: Gradle's `test…`/`…Test…` (testRuntimeClasspath, +// integrationTestCompileClasspath, debugUnitTestRuntimeClasspath), sbt's +// test and it configurations. +const TEST_CONFIG = /^test|Test|^it(?:-internal)?$/ +const SBT_TEST_CONFIG = /^(?:test|it)(?:-internal)?$/ + +// Lower ranks go first; an unrecognised configuration counts as tooling. +function classpathRank(config: string): number { + const n = config.toLowerCase() + // Gradle's legacy `runtime`/`compile` (before Gradle 7) are a subset of the + // matching `…Classpath` configuration, so they rank just behind it. + const legacy = n.endsWith('runtime') || n.endsWith('compile') ? 0.5 : 0 + const runtime = n.endsWith('runtimeclasspath') || n.endsWith('runtime') + const compile = n.endsWith('compileclasspath') || n.endsWith('compile') + if (TEST_CONFIG.test(config)) { + if (runtime || SBT_TEST_CONFIG.test(n)) { + return 4 + legacy + } + return compile ? 5 + legacy : 6 + } + if (n === 'runtimeclasspath' || n === 'runtime' || n === 'runtime-internal') { + return legacy + } + if (n === 'compileclasspath' || n === 'compile' || n === 'compile-internal') { + return 1 + legacy + } + if (runtime) { + return 2 + legacy + } + if (compile || n === 'provided' || n === 'optional') { + return 3 + legacy + } + return 6 +} + +// The JVM loads a class from the first classpath entry that has it, so the +// first root by rank to resolve a module fixes its version. +function deriveClasspath(roots: PerRoot[]): string[] { + const ranked = [...roots].sort( + (a, b) => classpathRank(a.config) - classpathRank(b.config), ) + const versionOf = new Map() + const out = new Set() + for (const { nodes } of ranked) { + for (const { 0: id, 1: node } of nodes) { + const module = node.project + ? `project ${node.project}` + : `${node.coord.group}:${node.coord.name}` + let version = versionOf.get(module) + if (version === undefined) { + version = node.coord.version + versionOf.set(module, version) + } + if (version === node.coord.version) { + out.add(id) + } + } + } + return [...out] +} + +// A project's classpath as its build tool states it, by component id. +function statedClasspath( + projectKey: string, + stated: string[], + roots: RawRoot[], +): string[] { + const ids = new Map() + for (const r of roots) { + for (const { 0: coordId, 1: n } of r.nodes) { + ids.set(coordId, componentId(coordId, n.project)) + } + } + const cp = new Set() + for (const coordId of stated) { + const id = ids.get(coordId) + if (id === undefined) { + throw new Error( + `The classpath of ${projectKey} names ${coordId}, which none of its resolution roots resolved`, + ) + } + cp.add(id) + } + return [...cp] +} + +function buildClasspaths( + parsed: ParsedRecords, + perRoot: Map, +): Map { + const rootIdsByProject = new Map() + for (const { 0: rootId, 1: r } of parsed.roots) { + let list = rootIdsByProject.get(r.projectKey) + if (!list) { + list = [] + rootIdsByProject.set(r.projectKey, list) + } + list.push(rootId) + } + const out = new Map() + for (const projectKey of parsed.projects.keys()) { + const rootIds = rootIdsByProject.get(projectKey) ?? [] + const stated = parsed.classpaths.get(projectKey) + out.set( + projectKey, + stated + ? statedClasspath( + projectKey, + stated, + rootIds.flatMap(id => parsed.roots.get(id) ?? []), + ) + : deriveClasspath(rootIds.flatMap(id => perRoot.get(id) ?? [])), + ) + } + return out } function buildArtifactPaths( @@ -413,6 +530,7 @@ function buildArtifactPaths( projects: RawProject[], excludedProjects: RawProject[], graphs: Map, + classpaths: Map, fileExists: (path: string) => boolean, ): ResolvedArtifactPaths { const pathsById: ResolvedArtifactPaths['pathsById'] = new Map() @@ -454,8 +572,13 @@ function buildArtifactPaths( } return { pathsById, - classpathByProject: byProject(projects, graphs, g => g.children.keys()), - directDependenciesByProject: byProject(projects, graphs, g => g.direct), + classpathByProject: classpaths, + directDependenciesByProject: new Map( + projects.map(p => { + const g = graphs.get(p.projectKey) + return [p.projectKey, g ? [...g.direct].sort() : []] + }), + ), } } diff --git a/src/commands/manifest/scripts/assemble.test.mts b/src/commands/manifest/scripts/assemble.test.mts index b78766fc55..7bb1d7fc47 100644 --- a/src/commands/manifest/scripts/assemble.test.mts +++ b/src/commands/manifest/scripts/assemble.test.mts @@ -100,7 +100,7 @@ describe('records → assemble → sidecar', () => { dependencies: ['com.example:bom:2.0', 'com.example:lib:jar:1.0'], targets: ['/abs/app/build/classes'], sources: ['/abs/app/src/main/java'], - classpath: ['com.example:bom:2.0', 'com.example:lib:jar:1.0'], + classpath: ['com.example:lib:jar:1.0', 'com.example:bom:2.0'], }, ]) @@ -262,11 +262,11 @@ describe('records → assemble → sidecar', () => { p, ]), ) - expect(byName.get('a')?.classpath).toEqual(['g:dep:jar:1', 'g:lib:jar:1']) + expect(byName.get('a')?.classpath).toEqual(['g:lib:jar:1', 'g:dep:jar:1']) expect(byName.get('b')?.classpath).toEqual([ + 'g:lib:jar:1', 'g:dep:jar:2', 'g:junit:jar:4', - 'g:lib:jar:1', ]) expect(byName.get('b')?.dependencies).toEqual([ 'g:junit:jar:4', @@ -544,6 +544,148 @@ describe('records → assemble → sidecar', () => { }) }) +function classpath(records: string[], project: string): string[] | undefined { + return assembleFacts(parseRecords(records.join('\n')), { + fileExists: () => true, + }).artifactPaths.classpathByProject.get(project) +} + +describe('classpath', () => { + it("follows each root's order, runtime before compile-only, test and tooling roots", () => { + expect( + classpath( + [ + 'meta\tgradle\t8.10\t17', + 'project\t:app\tg\tapp\t1\tapp', + 'root\t0\t:app\tannotationProcessor\t0', + 'node\t0\tg:proc:jar:1\tg\tproc\t1\tjar\t\t1', + 'root\t1\t:app\tcompileClasspath\t1', + 'node\t1\tg:api:jar:1\tg\tapi\t1\tjar\t\t1', + 'node\t1\tg:servlet:jar:3\tg\tservlet\t3\tjar\t\t1', + 'root\t2\t:app\truntimeClasspath\t1', + 'node\t2\tg:z:jar:1\tg\tz\t1\tjar\t\t1', + 'node\t2\tg:api:jar:1\tg\tapi\t1\tjar\t\t1', + 'root\t3\t:app\ttestRuntimeClasspath\t0', + 'node\t3\tg:junit:jar:4\tg\tjunit\t4\tjar\t\t1', + 'node\t3\tg:z:jar:1\tg\tz\t1\tjar\t\t1', + ], + ':app', + ), + ).toEqual([ + 'g:z:jar:1', + 'g:api:jar:1', + 'g:servlet:jar:3', + 'g:junit:jar:4', + 'g:proc:jar:1', + ]) + }) + + it('keeps the version of a module the highest-ranked root resolved, with its other artifacts', () => { + expect( + classpath( + [ + 'meta\tgradle\t8.10\t17', + 'project\t:app\tg\tapp\t1\tapp', + 'root\t0\t:app\ttestRuntimeClasspath\t0', + 'node\t0\tg:guava:jar:32\tg\tguava\t32\tjar\t\t1', + 'node\t0\tg:netty:jar:linux:4.1\tg\tnetty\t4.1\tjar\tlinux\t1', + 'root\t1\t:app\truntimeClasspath\t1', + 'node\t1\tg:guava:jar:30\tg\tguava\t30\tjar\t\t1', + 'node\t1\tg:netty:jar:4.1\tg\tnetty\t4.1\tjar\t\t1', + ], + ':app', + ), + ).toEqual(['g:guava:jar:30', 'g:netty:jar:4.1', 'g:netty:jar:linux:4.1']) + }) + + it("ranks Gradle's legacy runtime and compile behind the classpath configurations", () => { + expect( + classpath( + [ + 'meta\tgradle\t6.9\t11', + 'project\t:app\tg\tapp\t1\tapp', + 'root\t0\t:app\tcompile\t1', + 'node\t0\tg:foo:jar:1\tg\tfoo\t1\tjar\t\t1', + 'root\t1\t:app\tcompileClasspath\t1', + 'node\t1\tg:bar:jar:1\tg\tbar\t1\tjar\t\t1', + 'root\t2\t:app\truntime\t1', + 'node\t2\tg:foo:jar:1\tg\tfoo\t1\tjar\t\t1', + 'root\t3\t:app\truntimeClasspath\t1', + 'node\t3\tg:bar:jar:1\tg\tbar\t1\tjar\t\t1', + 'node\t3\tg:foo:jar:2\tg\tfoo\t2\tjar\t\t0', + 'edge\t3\tg:bar:jar:1\tg:foo:jar:2', + ], + ':app', + ), + ).toEqual(['g:bar:jar:1', 'g:foo:jar:2']) + }) + + it('tells test configurations by name segment, sbt it among them', () => { + expect( + classpath( + [ + 'meta\tgradle\t8.10\t17', + 'project\t:app\tg\tapp\t1\tapp', + 'root\t0\t:app\ttestRuntimeClasspath\t0', + 'node\t0\tg:x:jar:3\tg\tx\t3\tjar\t\t1', + 'root\t1\t:app\tlatestRuntimeClasspath\t1', + 'node\t1\tg:x:jar:2\tg\tx\t2\tjar\t\t1', + ], + ':app', + ), + ).toEqual(['g:x:jar:2']) + expect( + classpath( + [ + 'meta\tsbt\t1.9.9\t17', + 'project\tapp\tg\tapp\t1\tapp', + 'root\t0\tapp\tscala-tool\t0', + 'node\t0\tg:x:jar:2\tg\tx\t2\tjar\t\t1', + 'root\t1\tapp\tit\t0', + 'node\t1\tg:x:jar:1\tg\tx\t1\tjar\t\t1', + ], + 'app', + ), + ).toEqual(['g:x:jar:1']) + }) + + it('takes a classpath the build tool states as it is, by component id', () => { + expect( + classpath( + [ + 'meta\tmaven\t3.9.6\t17', + 'project\tg:app:1\tg\tapp\t1\tapp', + 'project\tg:lib:1\tg\tlib\t1\tlib', + 'root\t0\tg:app:1\tcompile\t1', + 'node\t0\tg:z:jar:1\tg\tz\t1\tjar\t\t1', + 'node\t0\tg:lib:1\tg\tlib\t1\t\t\t1\tg:lib:1', + 'root\t1\tg:app:1\ttest\t0', + 'node\t1\tg:bcpkix:jar:1\tg\tbcpkix\t1\tjar\t\t1', + 'classpath\tg:app:1\tg:bcpkix:jar:1', + 'classpath\tg:app:1\tg:lib:1', + 'classpath\tg:app:1\tg:z:jar:1', + ], + 'g:app:1', + ), + ).toEqual(['g:bcpkix:jar:1', 'g:lib:1', 'g:z:jar:1']) + }) + + it('rejects a stated classpath entry that no root of the project resolved', () => { + expect(() => + classpath( + [ + 'meta\tmaven\t3.9.6\t17', + 'project\tg:app:1\tg\tapp\t1\tapp', + 'root\t0\tg:app:1\tcompile\t1', + 'node\t0\tg:z:jar:1\tg\tz\t1\tjar\t\t1', + 'classpath\tg:app:1\tg:y:jar:1', + ], + 'g:app:1', + ), + ).toThrow(/names g:y:jar:1/) + }) +}) + describe('parseRecords', () => { it('reads the build root the build reports', () => { expect( diff --git a/src/commands/manifest/scripts/facts.mts b/src/commands/manifest/scripts/facts.mts index 6c48d539cd..97f8cfb786 100644 --- a/src/commands/manifest/scripts/facts.mts +++ b/src/commands/manifest/scripts/facts.mts @@ -64,8 +64,7 @@ export type SocketFactsSbomProject = AnyPURL & { // did not resolve; `sources` = module source roots. export type ResolvedArtifactPaths = { pathsById: Map - // Component ids on each project's resolved classpath (union over its - // configurations), keyed by project id. + // Component ids on each project's classpath, in order, keyed by project id. classpathByProject: Map // Component ids of each project's direct dependencies, keyed by project id. directDependenciesByProject: Map diff --git a/src/commands/manifest/scripts/maven-extension/src/main/java/tech/coana/socket/SocketFactsRecordsEngine.java b/src/commands/manifest/scripts/maven-extension/src/main/java/tech/coana/socket/SocketFactsRecordsEngine.java index 93c8ad272f..45eb890739 100644 --- a/src/commands/manifest/scripts/maven-extension/src/main/java/tech/coana/socket/SocketFactsRecordsEngine.java +++ b/src/commands/manifest/scripts/maven-extension/src/main/java/tech/coana/socket/SocketFactsRecordsEngine.java @@ -160,11 +160,13 @@ public void run(MavenSession session, List reactor, File rootDir, // A wholly excluded reactor module is not resolved (matches the project-record skip above). if (SocketSupport.isExcludedDir(excludeBase, module.getBasedir().toPath(), excludes)) continue; Map nodes = new LinkedHashMap<>(); + ClasspathOrder order = new ClasspathOrder(); Set directIds = new HashSet<>(); // Which direct dependencies are prod-scoped: seeds the prod/dev root split (see emitModuleRoots). Set directProdIds = new HashSet<>(); - collectModule(session, module, passingScopes, reactorGavs, populateGavs, opts, nodes, directIds, directProdIds, failures, unscannable); + collectModule(session, module, passingScopes, reactorGavs, populateGavs, opts, nodes, order, directIds, directProdIds, failures, unscannable); rootIdx = emitModuleRoots(lines, rootIdx, projectKey(module), nodes, directIds, directProdIds); + emitClasspath(lines, projectKey(module), order); } for (Failure f : failures) rec(lines, "failure", f.coord, f.detail, f.config, f.project); @@ -189,6 +191,7 @@ private void collectModule( Set populateGavs, Options opts, Map nodes, + ClasspathOrder order, Set directIds, Set directProdIds, Set failures, @@ -299,7 +302,7 @@ private void collectModule( if (graph == null) return; Set visited = new HashSet<>(); for (DependencyNode child : graph.getChildren()) { - String id = visit(child, passingScopes, reactorGavs, unresolved, opts, nodes, visited); + String id = visit(child, passingScopes, reactorGavs, unresolved, opts, nodes, order, visited); if (id != null) { directIds.add(id); // A direct dependency's effective scope is its declared scope (Aether's scope selector keeps @@ -400,6 +403,7 @@ private String visit( Set unresolved, Options opts, Map nodes, + ClasspathOrder order, Set visited) { Dependency dep = dn.getDependency(); Artifact artifact = dn.getArtifact(); @@ -427,6 +431,7 @@ private String visit( ? upsert(nodes, id, artifact.getGroupId(), artifact.getArtifactId(), "", "", version, gav) : upsert(nodes, id, artifact.getGroupId(), artifact.getArtifactId(), type == null ? "" : type, classifier == null ? "" : classifier, version, ""); + ("test".equals(scope) ? order.test : order.main).add(id); // Maven wrote each accepted node's resolved file back onto the node; a reactor module reports its // own dirs through its `project` record instead of a `file` record. if (!internal && opts.withFiles) { @@ -435,7 +440,7 @@ private String visit( } for (DependencyNode child : dn.getChildren()) { - String childId = visit(child, passingScopes, reactorGavs, unresolved, opts, nodes, visited); + String childId = visit(child, passingScopes, reactorGavs, unresolved, opts, nodes, order, visited); if (childId != null) node.children.add(childId); } return id; @@ -503,6 +508,16 @@ private int emitModuleRoots( return rootIdx; } + // Maven builds every classpath of a module from one graph, in the pre-order its own + // RepositoryUtils.toArtifacts walks, filtered by each artifact's effective scope. The test scope goes + // last, so that a test-only artifact never shadows one main code runs with. + private static void emitClasspath(List lines, String projectKey, ClasspathOrder order) { + for (String id : order.main) rec(lines, "classpath", projectKey, id); + for (String id : order.test) { + if (!order.main.contains(id)) rec(lines, "classpath", projectKey, id); + } + } + // Every node id reachable from `seeds` by following retained child edges. private static Set reachable(Map nodes, Set seeds) { Set seen = new HashSet<>(); @@ -873,6 +888,13 @@ public int hashCode() { } } + // Node ids in walk order, by whether the artifact reached them with the test scope. A sibling module's + // variants share a node, and can reach it with different scopes. + private static final class ClasspathOrder { + final Set main = new LinkedHashSet<>(); + final Set test = new LinkedHashSet<>(); + } + private static final class Node { final String id; final String groupId; diff --git a/src/commands/manifest/scripts/records.mts b/src/commands/manifest/scripts/records.mts index 2e99cd630a..5089d5fcb3 100644 --- a/src/commands/manifest/scripts/records.mts +++ b/src/commands/manifest/scripts/records.mts @@ -17,6 +17,7 @@ import type { ResolutionFailure, Unscannable } from './resolution-report.mts' // root rootId projectKey config prod(0|1) // node rootId coordId group name version ext classifier direct(0|1) project // edge rootId parentCoordId childCoordId +// classpath projectKey coordId (in order; see below) // file rootId coordId path (--with-files only) // unfetched rootId coordId (--with-files only; requested artifact not fetched) // scanned config @@ -29,6 +30,9 @@ import type { ResolutionFailure, Unscannable } from './resolution-report.mts' // A `root` is one (subproject, configuration) resolution root; `coordId` is the // coordinate key (`group:name:ext:classifier:version`, empty segments dropped), // used opaquely as the per-root node key. Unknown tags are ignored. +// +// `node` records follow the root's classpath order. `classpath` records state a +// project's whole classpath (Maven); without them the assembler derives it. export type RawCoord = { group: string @@ -85,6 +89,8 @@ export type ParsedRecords = { // the kept projects depending on them. excludedProjects: Map roots: Map + // Keyed by projectKey; coordIds of the project's roots' nodes, in order. + classpaths: Map scannedConfigs: string[] failures: ResolutionFailure[] unscannable: Unscannable[] @@ -121,6 +127,7 @@ export function parseRecords(text: string): ParsedRecords { projects: new Map(), excludedProjects: new Map(), roots: new Map(), + classpaths: new Map(), scannedConfigs: [], failures: [], unscannable: [], @@ -238,6 +245,18 @@ export function parseRecords(text: string): ParsedRecords { } break } + case 'classpath': { + const key = f[1] ?? '' + let cp = result.classpaths.get(key) + if (!cp) { + cp = [] + result.classpaths.set(key, cp) + } + if (f[2]) { + cp.push(f[2]) + } + break + } case 'unfetched': { const node = root(f[1] ?? '').nodes.get(f[2] ?? '') if (node) { diff --git a/src/commands/manifest/scripts/sidecar.mts b/src/commands/manifest/scripts/sidecar.mts index a19b542126..136934db90 100644 --- a/src/commands/manifest/scripts/sidecar.mts +++ b/src/commands/manifest/scripts/sidecar.mts @@ -21,8 +21,9 @@ export type SidecarProjectEntry = Omit & { sources?: string[] | undefined // Ids of this facts file's components[] the project depends on directly. dependencies: string[] - // Ids of this facts file's components[] forming the project's full - // transitive classpath across all its configurations. + // Ids of this facts file's components[] on the project's classpath: one + // version per module, main code's entries before test-only and tooling ones. + // When artifact paths were resolved, in the order the JVM searches them. classpath: string[] } diff --git a/src/commands/manifest/scripts/socket-facts.init.gradle b/src/commands/manifest/scripts/socket-facts.init.gradle index 7558a68c60..11d0b7ee79 100644 --- a/src/commands/manifest/scripts/socket-facts.init.gradle +++ b/src/commands/manifest/scripts/socket-facts.init.gradle @@ -569,22 +569,111 @@ allprojects { project -> directIds.addAll(direct) } + // A lenient view of this config's artifacts, limited to the modules the walk fetched files for (and + // which verification kept), plus project outputs when `withProjects`. + def fetchedView = { cfg, withProjects -> + def fetched = paths.keySet().findAll { id -> nodes[id]?.project == null }.collect { id -> gavOf(nodes[id].coord) } as Set + cfg.incoming.artifactView { + lenient(true) + componentFilter { cid -> + cid instanceof org.gradle.api.artifacts.component.ModuleComponentIdentifier ? + fetched.contains("${cid.group}:${cid.module}:${cid.version}".toString()) : withProjects + } + } + } + + // Node ids in the order the fetched-artifact view lists them; null before Gradle 4.0, which has no + // lenient artifact view. + def orderFromView = { cfg -> + def view + try { + view = fetchedView(cfg, true) + } catch (MissingMethodException e) { + return null + } + def idsOfProject = [:] + nodes.each { id, n -> if (n.project != null) { idsOfProject.get(n.project, []) << id } } + def idsByFile = [:] + paths.each { id, files -> files.each { f -> idsByFile.get(f, []) << id } } + def order = [] + view.artifacts.each { a -> + def cid = a.id.componentIdentifier + if (cid instanceof org.gradle.api.artifacts.component.ProjectComponentIdentifier) { + def b = cid.metaClass.respondsTo(cid, 'getBuild') ? cid.build : null + def buildPath = b == null ? ':' : b.metaClass.respondsTo(b, 'getBuildPath') ? b.buildPath : b.name + // An included build's project paths can repeat this build's; its artifacts are files. + if (buildPath == ':') { + order.addAll(idsOfProject[cid.projectPath] ?: []) + return + } + } + order.addAll(idsByFile[a.file.absolutePath] ?: []) + } + order + } + + // Before Gradle 4.0. The legacy listing cannot leave out the artifacts a populateFilesFor scope + // excludes, so a scoped run keeps the walk's order. + def orderFromLegacy = { lenientCfg -> + if (populateGavs != null) { + return [] + } + def artifacts + try { + artifacts = lenientCfg.artifacts + } catch (MissingPropertyException e) { + artifacts = lenientCfg.getArtifacts({ true } as org.gradle.api.specs.Spec) + } + artifacts.collect { a -> + def mv = a.moduleVersion.id + def projPath = projectByGav["${mv.group ?: ''}:${mv.name}:${mv.version ?: ''}".toString()] + coordId([groupId: mv.group ?: '', artifactId: mv.name, version: mv.version ?: '', + classifier: a.classifier ?: '', ext: effExt(projPath, a.extension)]) + } + } + + // Puts this config's nodes in the order Gradle lists its artifacts: its classpath order, which no + // walk of the dependency graph reproduces. Listing them fetches their files, so only a --with-files + // run reads it, over the artifacts it fetched; a plain run keeps the walk's order. + def orderByArtifacts = { cfg, lenientCfg -> + if (!withFiles) { + return + } + def order + try { + order = orderFromView(cfg) + if (order == null) { + order = orderFromLegacy(lenientCfg) + } + } catch (Exception e) { + println "[socket-facts] could not order the classpath of ${configName}: ${rawDetail(e).readLines().find()}" + return + } + def listed = new LinkedHashSet(order.findAll { id -> nodes.containsKey(id) }) + if (listed.isEmpty()) { + return + } + // A node the listing leaves out (a pom-only module, an artifact this run did not fetch) keeps its + // place in the walk; the listed nodes fill the remaining places in classpath order. + def next = listed.iterator() + def ordered = [:] + nodes.each { id, node -> + def slot = listed.contains(id) ? next.next() : id + ordered[slot] = nodes[slot] + } + nodes.clear() + nodes.putAll(ordered) + } + // Drops the files of this config's artifacts that failed verification and reports them. def checkVerification = { cfg -> def external = paths.keySet().findAll { id -> nodes[id].project == null } if (external.isEmpty()) { return } - def fetched = external.collect { id -> gavOf(nodes[id].coord) } as Set def message = null try { - cfg.incoming.artifactView { - lenient(true) - componentFilter { id -> - id instanceof org.gradle.api.artifacts.component.ModuleComponentIdentifier && - fetched.contains("${id.group}:${id.module}:${id.version}".toString()) - } - }.artifacts.each { it.file } + fetchedView(cfg, false).artifacts.each { it.file } } catch (Exception e) { message = verificationMessage(e) if (message == null) { @@ -714,6 +803,7 @@ allprojects { project -> if (verifyArtifacts) { checkVerification(cfg) } + orderByArtifacts(cfg, lenient) walked = true } catch (Exception e) { // A config-LEVEL resolution failure (a resolutionStrategy/beforeResolve rule that throws, diff --git a/src/commands/manifest/scripts/test/gradle-compat/order/build.gradle b/src/commands/manifest/scripts/test/gradle-compat/order/build.gradle new file mode 100644 index 0000000000..24a54c14bd --- /dev/null +++ b/src/commands/manifest/scripts/test/gradle-compat/order/build.gradle @@ -0,0 +1,24 @@ +// Ordering fixture for ../../socket-facts.init.gradle: `a` pulls `a1`, so a walk of the dependency +// graph meets a, a1, b while Gradle puts a's dependencies after its own siblings on the classpath. +// The `printClasspath` task writes the order Gradle itself builds the runtime classpath in. +apply plugin: 'java' +group = 'demo' +version = '1.0.0' + +repositories { + maven { url file('localrepo').toURI() } +} + +def modern = org.gradle.util.GradleVersion.current() >= org.gradle.util.GradleVersion.version('3.4') +def runtimeName = modern ? 'runtimeClasspath' : 'runtime' +dependencies { + add(modern ? 'implementation' : 'compile', 'demo.lib:a:1.0') + add(modern ? 'implementation' : 'compile', 'demo.lib:b:1.0') +} + +tasks.create("printClasspath") { + doLast { + file(project.property('classpathFile')).text = + runtimeName + '\n' + configurations.getByName(runtimeName).files.collect { it.name }.join('\n') + '\n' + } +} diff --git a/src/commands/manifest/scripts/test/gradle-compat/order/settings.gradle b/src/commands/manifest/scripts/test/gradle-compat/order/settings.gradle new file mode 100644 index 0000000000..0b8b772d2b --- /dev/null +++ b/src/commands/manifest/scripts/test/gradle-compat/order/settings.gradle @@ -0,0 +1 @@ +rootProject.name = 'gradle-compat-order' diff --git a/src/commands/manifest/scripts/test/gradle-compat/smoke-test-order.sh b/src/commands/manifest/scripts/test/gradle-compat/smoke-test-order.sh new file mode 100755 index 0000000000..817f26818b --- /dev/null +++ b/src/commands/manifest/scripts/test/gradle-compat/smoke-test-order.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Run socket-facts.init.gradle with --with-files on the ordering fixture and assert the runtime +# classpath root lists its nodes in the order Gradle itself builds that classpath in, which the +# fixture's own `printClasspath` task reports. A plain graph walk would give a different order. +# +# Usage: smoke-test-order.sh /path/to/gradle +set -euo pipefail +HERE="$(cd "$(dirname "$0")" && pwd)" +GRADLE="${1:?usage: smoke-test-order.sh }" +INIT="$HERE/../../socket-facts.init.gradle" +PROJECT="$HERE/order" +RECORDS="$PROJECT/records.tsv" +CLASSPATH="$PROJECT/classpath.txt" +# shellcheck source=SCRIPTDIR/../compat-cache.sh +. "$HERE/../compat-cache.sh" +GUH="$SOCKET_COMPAT_CACHE/gradle-home" + +bash "$HERE/../make-stub-repo.sh" "$PROJECT/localrepo" 'demo.lib:a:1.0+demo.lib:a1:1.0' 'demo.lib:a1:1.0' 'demo.lib:b:1.0' +rm -rf "$GUH" "$RECORDS" "$CLASSPATH" "$PROJECT/.gradle" "$PROJECT/build" + +echo "+ $("$GRADLE" --version 2>/dev/null | sed -n 's/^Gradle //p' | head -1) (order)" +( cd "$PROJECT" && "$GRADLE" --no-daemon --offline -g "$GUH" --init-script "$INIT" \ + -Psocket.withFiles=true -Psocket.recordsFile="$RECORDS" -PclasspathFile="$CLASSPATH" \ + socketFacts printClasspath -q ) + +python3 - "$RECORDS" "$CLASSPATH" <<'PY' +import sys +rows = [l.rstrip('\n').split('\t') for l in open(sys.argv[1]) if l.strip()] +config, *jars = [l.strip() for l in open(sys.argv[2]) if l.strip()] +root = next((r[1] for r in rows if r[0] == 'root' and r[3] == config), None) +names = [f"{r[4]}-{r[5]}.jar" for r in rows if r[0] == 'node' and r[1] == root] +if root is None: + print(f"FAIL: no {config} root"); sys.exit(1) +if names != jars: + print(f"FAIL: {config} nodes {names} != Gradle's classpath order {jars}"); sys.exit(1) +print(f"PASS (order): {config} nodes in Gradle's classpath order {jars}") +PY diff --git a/src/commands/manifest/scripts/test/maven-compat/assert-classpath.py b/src/commands/manifest/scripts/test/maven-compat/assert-classpath.py new file mode 100644 index 0000000000..a6062dff0f --- /dev/null +++ b/src/commands/manifest/scripts/test/maven-compat/assert-classpath.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +# Assert the `classpath` records the Coana Maven core extension emitted for the smoke project's app +# modules: the order Maven builds their classpaths in (a pre-order walk of the resolved graph, filtered +# by scope, the test scope moved last). app keeps demo.ext:libdep, which Maven leaves only under lib's +# test-jar node while the walk reaches lib's jar node first; app2 reaches lib's node through the +# test-jar first and still puts lib on the main classpath. Matches on groupId:artifactId, so versions +# can change freely. +import sys + +rows = [l.rstrip('\n').split('\t') for l in open(sys.argv[1]) if l.strip()] +root_project = {r[1]: r[2] for r in rows if r[0] == 'root'} + +EXPECTED = { + 'demo:app:1.0': [ + 'demo:mid', 'demo:lib', 'demo.ext:tool', 'demo.ext:shared', 'demo.ext:conflict-prod', + 'demo.ext:libdep', 'demo.ext:harness', 'demo.ext:harness-core', 'demo.ext:conflict-test', + ], + 'demo:app2:1.0': ['demo.ext:libdep', 'demo:mid', 'demo:lib'], +} + +errors = [] +for project, expected in EXPECTED.items(): + nodes = {r[2] for r in rows if r[0] == 'node' and root_project.get(r[1]) == project} + classpath = [r[2] for r in rows if r[0] == 'classpath' and r[1] == project] + if sorted(classpath) != sorted(nodes): + errors.append(f"{project}: classpath {sorted(classpath)} != its nodes {sorted(nodes)}") + got = [':'.join(c.split(':')[:2]) for c in classpath] + if got != expected: + errors.append(f"{project}: classpath order {got} != {expected}") + +if errors: + print("FAIL:") + for e in errors: print(" -", e) + sys.exit(1) +print(f"PASS (classpath): {', '.join(EXPECTED)} in Maven's order, test scope last") diff --git a/src/commands/manifest/scripts/test/maven-compat/project/app2/pom.xml b/src/commands/manifest/scripts/test/maven-compat/project/app2/pom.xml new file mode 100644 index 0000000000..c953f5daaa --- /dev/null +++ b/src/commands/manifest/scripts/test/maven-compat/project/app2/pom.xml @@ -0,0 +1,26 @@ + + + 4.0.0 + + demo + root + 1.0 + + app2 + + + + demo + lib + 1.0 + test-jar + test + + + demo + mid + 1.0 + + + diff --git a/src/commands/manifest/scripts/test/maven-compat/project/app2/src/main/java/demo/App2.java b/src/commands/manifest/scripts/test/maven-compat/project/app2/src/main/java/demo/App2.java new file mode 100644 index 0000000000..0bd66ee50a --- /dev/null +++ b/src/commands/manifest/scripts/test/maven-compat/project/app2/src/main/java/demo/App2.java @@ -0,0 +1,4 @@ +package demo; + +public class App2 { +} diff --git a/src/commands/manifest/scripts/test/maven-compat/project/pom.xml b/src/commands/manifest/scripts/test/maven-compat/project/pom.xml index c9dc686a95..5b4eceb6ec 100644 --- a/src/commands/manifest/scripts/test/maven-compat/project/pom.xml +++ b/src/commands/manifest/scripts/test/maven-compat/project/pom.xml @@ -9,6 +9,7 @@ lib mid app + app2 8 diff --git a/src/commands/manifest/scripts/test/maven-compat/smoke-test-workspaces.sh b/src/commands/manifest/scripts/test/maven-compat/smoke-test-workspaces.sh index 116e871f15..7d9f83cfd1 100755 --- a/src/commands/manifest/scripts/test/maven-compat/smoke-test-workspaces.sh +++ b/src/commands/manifest/scripts/test/maven-compat/smoke-test-workspaces.sh @@ -45,10 +45,10 @@ if tool != 'maven': errors.append(f"meta tool {tool!r} != 'maven'") if build_root != sys.argv[2]: errors.append(f"buildRoot {build_root!r} != {sys.argv[2]!r}") -expected_build_files = {'pom.xml', 'lib/pom.xml', 'mid/pom.xml', 'app/pom.xml'} +expected_build_files = {'pom.xml', 'lib/pom.xml', 'mid/pom.xml', 'app/pom.xml', 'app2/pom.xml'} if build_files != expected_build_files: errors.append(f"projectBuild files {sorted(build_files)} != {sorted(expected_build_files)}") -expected = {'root', 'lib', 'mid', 'app'} +expected = {'root', 'lib', 'mid', 'app', 'app2'} if set(projects) != expected: errors.append(f"expected reactor modules {sorted(expected)}, got {sorted(projects)}") for artifact in expected & set(projects): diff --git a/src/commands/manifest/scripts/test/maven-compat/smoke-test.sh b/src/commands/manifest/scripts/test/maven-compat/smoke-test.sh index be8dac3f40..a17699562b 100755 --- a/src/commands/manifest/scripts/test/maven-compat/smoke-test.sh +++ b/src/commands/manifest/scripts/test/maven-compat/smoke-test.sh @@ -8,8 +8,8 @@ # inter-module edge lines up with its `project` record); # - resolves a scope conflict so the retained node stays reachable from its parent root; # - every component is reachable from a direct dependency of its own root (no orphaned components); -# - keeps the dependencies Maven leaves only under a sibling module's test-jar (app reaches -# demo.ext:libdep under lib's test-jar node, after lib's jar node through mid); +# - states each module's classpath in Maven's order, test scope last, keeping the dependencies Maven +# leaves only under a sibling module's test-jar (assert-classpath.py); # - materializes resolved external jars under -Dsocket.withFiles; # - scopes that materialization to -Dsocket.populateFilesFor (a newline-delimited GAV file). # @@ -55,14 +55,7 @@ echo "+ $("$MVN" -v 2>/dev/null | head -1)" compile ) python3 "$HERE/assert-records.py" "$RECORDS" -python3 - "$RECORDS" <<'PY' -import sys -rows = [l.rstrip('\n').split('\t') for l in open(sys.argv[1]) if l.strip()] -app = {r[1] for r in rows if r[0] == 'root' and r[2] == 'demo:app:1.0'} -if not any(r[0] == 'node' and r[1] in app and r[2].startswith('demo.ext:libdep:') for r in rows): - print("FAIL: demo.ext:libdep missing from app"); sys.exit(1) -print("PASS: demo.ext:libdep in app through lib's test-jar") -PY +python3 "$HERE/assert-classpath.py" "$RECORDS" python3 "$HERE/../assert-reachability.py" "$RECORDS" # Second run: scope --with-files to a single GAV and assert ONLY that artifact is materialized. diff --git a/src/commands/manifest/scripts/test/run-compat.sh b/src/commands/manifest/scripts/test/run-compat.sh index eaa8d75125..ad9cabe960 100755 --- a/src/commands/manifest/scripts/test/run-compat.sh +++ b/src/commands/manifest/scripts/test/run-compat.sh @@ -66,6 +66,7 @@ run_gradle() { bash "$HERE/gradle-compat/smoke-test-workspaces.sh" "$dir/bin/gradle" bash "$HERE/gradle-compat/smoke-test-exclude-base.sh" "$dir/bin/gradle" bash "$HERE/gradle-compat/smoke-test-failures.sh" "$dir/bin/gradle" + bash "$HERE/gradle-compat/smoke-test-order.sh" "$dir/bin/gradle" done } From 912d2054397d9ea9a035e0981df688421edb9c19 Mon Sep 17 00:00:00 2001 From: Jeppe Fredsgaard Blaabjerg Date: Sun, 1 Nov 2026 10:47:26 +0100 Subject: [PATCH 2/2] feat(manifest): give each output set its own sidecar classpath A project now gets one runtime classpath per output set (main, test, testFixtures, KMP targets, Android variants), each one tool resolution in the order the JVM searches it, instead of one classpath merged across configurations. Test-only versions stay on the test classpath rather than being dropped. - Project sources and targets are split by output set. - A classpath entry is an external artifact or a build project's output set, so a dependency on a sibling's test-jar, testFixtures or test->test brings in that code and nothing else. - Each set's classpath starts with the project's own code and the output sets the tool wires into it (test holds main). - Compile-only entries (provided, compileOnly, system) follow the runtime ones, at their declared version. Producers state the mapping from their own model: Maven's scopes, Gradle source sets, Kotlin compilations and AGP variants, and sbt's classpath configurations. --- .../manifest/run-manifest-facts.test.mts | 3 +- src/commands/manifest/scripts/assemble.mts | 224 ++++++++------ .../manifest/scripts/assemble.test.mts | 276 ++++++++++-------- src/commands/manifest/scripts/facts.mts | 27 +- .../socket/SocketFactsRecordsEngine.java | 84 ++++-- src/commands/manifest/scripts/records.mts | 91 ++++-- src/commands/manifest/scripts/sidecar.mts | 46 +-- .../manifest/scripts/sidecar.test.mts | 130 ++++++--- .../manifest/scripts/socket-facts.init.gradle | 208 ++++++++++--- .../scripts/socket-facts.plugin.scala | 95 ++++-- .../test/maven-compat/assert-classpath.py | 46 +-- .../scripts/test/maven-compat/smoke-test.sh | 2 +- 12 files changed, 807 insertions(+), 425 deletions(-) diff --git a/src/commands/manifest/run-manifest-facts.test.mts b/src/commands/manifest/run-manifest-facts.test.mts index 45b45cf77f..543bb25420 100644 --- a/src/commands/manifest/run-manifest-facts.test.mts +++ b/src/commands/manifest/run-manifest-facts.test.mts @@ -34,7 +34,8 @@ function okResult(buildRoot: string): ManifestRunResult { }, artifactPaths: { pathsById: new Map(), - classpathByProject: new Map(), + outputsByProject: new Map(), + classpathsByProject: new Map(), directDependenciesByProject: new Map(), }, stderr: '', diff --git a/src/commands/manifest/scripts/assemble.mts b/src/commands/manifest/scripts/assemble.mts index e642b411b2..0a58617270 100644 --- a/src/commands/manifest/scripts/assemble.mts +++ b/src/commands/manifest/scripts/assemble.mts @@ -1,20 +1,24 @@ import { existsSync } from 'node:fs' -import { - type ResolvedArtifactPaths, - type SocketFactsSbom, - type SocketFactsSbomComponent, - type SocketFactsSbomDependency, - type SocketFactsSbomMetadata, - type SocketFactsSbomProject, -} from './facts.mts' import { moduleKey } from './resolution-report.mts' +import type { + ClasspathEntry, + OutputPaths, + ResolvedArtifactPaths, + SocketFactsSbom, + SocketFactsSbomComponent, + SocketFactsSbomDependency, + SocketFactsSbomMetadata, + SocketFactsSbomProject, +} from './facts.mts' + import type { ParsedRecords, RawCoord, RawProject, RawRoot, + StatedClasspathEntry, } from './records.mts' import type { ResolutionReport } from './resolution-report.mts' @@ -49,8 +53,9 @@ type SubprojectGraph = { type PerRoot = { projectKey: string - config: string prod: boolean + outputSet: string + role: RawRoot['role'] // In the root's classpath order. nodes: Map } @@ -60,6 +65,7 @@ type RootNode = { children: string[] direct: boolean project: string + outputSets: string[] targets: string[] unfetched: boolean } @@ -127,11 +133,17 @@ function buildPerRoot(parsed: ParsedRecords): Map { children: [], direct: false, project: n.project, + outputSets: [], targets: [], unfetched: false, } nodes.set(id, node) } + for (const set of n.outputSets) { + if (!node.outputSets.includes(set)) { + node.outputSets.push(set) + } + } node.direct ||= n.direct node.unfetched ||= n.unfetched node.targets.push(...n.targets) @@ -149,8 +161,9 @@ function buildPerRoot(parsed: ParsedRecords): Map { } out.set(rootId, { projectKey: r.projectKey, - config: r.config, prod: r.prod, + outputSet: r.outputSet, + role: r.role, nodes, }) } @@ -158,7 +171,7 @@ function buildPerRoot(parsed: ParsedRecords): Map { } // Components are merged by id across every resolution root; which ids belong -// to which subproject is kept separately (classpathByProject) for +// to which subproject is kept separately (classpathsByProject) for // reachability. function mergeById(perRoot: Map): Map { const finalNodes = new Map() @@ -408,50 +421,51 @@ function treeRoots( return roots } -// A test configuration: Gradle's `test…`/`…Test…` (testRuntimeClasspath, -// integrationTestCompileClasspath, debugUnitTestRuntimeClasspath), sbt's -// test and it configurations. -const TEST_CONFIG = /^test|Test|^it(?:-internal)?$/ -const SBT_TEST_CONFIG = /^(?:test|it)(?:-internal)?$/ - -// Lower ranks go first; an unrecognised configuration counts as tooling. -function classpathRank(config: string): number { - const n = config.toLowerCase() - // Gradle's legacy `runtime`/`compile` (before Gradle 7) are a subset of the - // matching `…Classpath` configuration, so they rank just behind it. - const legacy = n.endsWith('runtime') || n.endsWith('compile') ? 0.5 : 0 - const runtime = n.endsWith('runtimeclasspath') || n.endsWith('runtime') - const compile = n.endsWith('compileclasspath') || n.endsWith('compile') - if (TEST_CONFIG.test(config)) { - if (runtime || SBT_TEST_CONFIG.test(n)) { - return 4 + legacy - } - return compile ? 5 + legacy : 6 - } - if (n === 'runtimeclasspath' || n === 'runtime' || n === 'runtime-internal') { - return legacy - } - if (n === 'compileclasspath' || n === 'compile' || n === 'compile-internal') { - return 1 + legacy - } - if (runtime) { - return 2 + legacy - } - if (compile || n === 'provided' || n === 'optional') { - return 3 + legacy +// A dependency on a build project names the output sets it brings in; without +// one it is a plain dependency, which brings the project's main code. +function entriesOf( + id: string, + project: string, + outputSets: string[], +): ClasspathEntry[] { + if (!project) { + return [{ id }] } - return 6 + return (outputSets.length ? outputSets : ['main']).map(outputs => ({ + project: id, + outputs, + })) } -// The JVM loads a class from the first classpath entry that has it, so the -// first root by rank to resolve a module fixes its version. -function deriveClasspath(roots: PerRoot[]): string[] { - const ranked = [...roots].sort( - (a, b) => classpathRank(a.config) - classpathRank(b.config), +// The project's own code goes first, the way the build tools put it. +function ownEntries(project: RawProject, outputSet: string): ClasspathEntry[] { + return [outputSet, ...(project.outputSets.get(outputSet) ?? [])].map( + outputs => ({ project: project.projectKey, outputs }), ) +} + +function uniqueEntries(entries: ClasspathEntry[]): ClasspathEntry[] { + const seen = new Set() + return entries.filter(e => { + const key = 'id' in e ? e.id : `${e.project}\u0000${e.outputs}` + if (seen.has(key)) { + return false + } + seen.add(key) + return true + }) +} + +// The runtime classpath, then the entries only the compile classpath has +// (provided, compileOnly), in the version the runtime one resolved, if any. +function derivedClasspath(roots: PerRoot[]): ClasspathEntry[] { + const ordered = [ + ...roots.filter(r => r.role === 'runtime'), + ...roots.filter(r => r.role === 'compile'), + ] const versionOf = new Map() - const out = new Set() - for (const { nodes } of ranked) { + const out: ClasspathEntry[] = [] + for (const { nodes } of ordered) { for (const { 0: id, 1: node } of nodes) { const module = node.project ? `project ${node.project}` @@ -462,42 +476,33 @@ function deriveClasspath(roots: PerRoot[]): string[] { versionOf.set(module, version) } if (version === node.coord.version) { - out.add(id) + out.push(...entriesOf(id, node.project, node.outputSets)) } } } - return [...out] + return out } -// A project's classpath as its build tool states it, by component id. function statedClasspath( projectKey: string, - stated: string[], - roots: RawRoot[], -): string[] { - const ids = new Map() - for (const r of roots) { - for (const { 0: coordId, 1: n } of r.nodes) { - ids.set(coordId, componentId(coordId, n.project)) - } - } - const cp = new Set() - for (const coordId of stated) { - const id = ids.get(coordId) - if (id === undefined) { + stated: StatedClasspathEntry[], + projectOf: Map, +): ClasspathEntry[] { + return stated.flatMap(({ coordId, outputSets }) => { + const project = projectOf.get(coordId) + if (project === undefined) { throw new Error( `The classpath of ${projectKey} names ${coordId}, which none of its resolution roots resolved`, ) } - cp.add(id) - } - return [...cp] + return entriesOf(componentId(coordId, project), project, outputSets) + }) } function buildClasspaths( parsed: ParsedRecords, perRoot: Map, -): Map { +): Map> { const rootIdsByProject = new Map() for (const { 0: rootId, 1: r } of parsed.roots) { let list = rootIdsByProject.get(r.projectKey) @@ -507,20 +512,50 @@ function buildClasspaths( } list.push(rootId) } - const out = new Map() - for (const projectKey of parsed.projects.keys()) { + const out = new Map>() + for (const { 0: projectKey, 1: project } of parsed.projects) { const rootIds = rootIdsByProject.get(projectKey) ?? [] - const stated = parsed.classpaths.get(projectKey) - out.set( - projectKey, - stated - ? statedClasspath( - projectKey, - stated, - rootIds.flatMap(id => parsed.roots.get(id) ?? []), - ) - : deriveClasspath(rootIds.flatMap(id => perRoot.get(id) ?? [])), + // Every output set has a classpath, if only its own code. + const classpaths = new Map( + [...project.outputSets.keys()].map(name => [name, []]), ) + const stated = parsed.classpaths.get(projectKey) + if (stated) { + const projectOf = new Map() + for (const id of rootIds) { + for (const { 0: coordId, 1: n } of parsed.roots.get(id)?.nodes ?? []) { + projectOf.set(coordId, n.project) + } + } + for (const { 0: name, 1: entries } of stated) { + classpaths.set(name, statedClasspath(projectKey, entries, projectOf)) + } + } else { + const bySet = new Map() + for (const id of rootIds) { + const r = perRoot.get(id) + if (r?.outputSet && r.role) { + let roots = bySet.get(r.outputSet) + if (!roots) { + roots = [] + bySet.set(r.outputSet, roots) + } + roots.push(r) + } + } + for (const { 0: name, 1: roots } of bySet) { + if (roots.some(r => r.role === 'runtime')) { + classpaths.set(name, derivedClasspath(roots)) + } + } + } + for (const { 0: name, 1: entries } of classpaths) { + classpaths.set( + name, + uniqueEntries([...ownEntries(project, name), ...entries]), + ) + } + out.set(projectKey, classpaths) } return out } @@ -530,7 +565,7 @@ function buildArtifactPaths( projects: RawProject[], excludedProjects: RawProject[], graphs: Map, - classpaths: Map, + classpaths: Map>, fileExists: (path: string) => boolean, ): ResolvedArtifactPaths { const pathsById: ResolvedArtifactPaths['pathsById'] = new Map() @@ -554,25 +589,28 @@ function buildArtifactPaths( pathsById.set( id, fn.resolutions.length && !complete.length - ? { sources: [] } - : { - sources: [], - targets: [...new Set(complete.flatMap(r => r.targets))].sort(), - }, + ? {} + : { targets: [...new Set(complete.flatMap(r => r.targets))].sort() }, ) } } // A project's own component shares its id, so this also covers dependency // edges onto a sibling project. + const outputsByProject = new Map>() for (const p of [...projects, ...excludedProjects]) { - pathsById.set(p.projectKey, { - sources: [...new Set(p.sources)].filter(fileExists).sort(), - targets: [...new Set(p.targets)].filter(fileExists).sort(), - }) + const sets = new Map() + for (const { 0: name, 1: o } of p.outputs) { + sets.set(name, { + sources: [...new Set(o.sources)].filter(fileExists).sort(), + targets: [...new Set(o.targets)].filter(fileExists).sort(), + }) + } + outputsByProject.set(p.projectKey, sets) } return { pathsById, - classpathByProject: classpaths, + outputsByProject, + classpathsByProject: classpaths, directDependenciesByProject: new Map( projects.map(p => { const g = graphs.get(p.projectKey) diff --git a/src/commands/manifest/scripts/assemble.test.mts b/src/commands/manifest/scripts/assemble.test.mts index 7bb1d7fc47..3be4eef28f 100644 --- a/src/commands/manifest/scripts/assemble.test.mts +++ b/src/commands/manifest/scripts/assemble.test.mts @@ -4,7 +4,7 @@ import { assembleFacts } from './assemble.mts' import { parseRecords } from './records.mts' import { accumulateSidecar, serializeSidecar } from './sidecar.mts' -import type { SocketFactsSbom } from './facts.mts' +import type { ClasspathEntry, SocketFactsSbom } from './facts.mts' import type { SidecarAccumulator } from './sidecar.mts' // Each project's tree, written out with component ids; a repeat on the current @@ -32,9 +32,9 @@ function trees(facts: SocketFactsSbom): Record { const RECORDS = [ 'meta\tgradle\t8.0\t17', 'project\t:app\tcom.example\tapp\t1.0\t/abs/app', - 'projectSrc\t:app\t/abs/app/src/main/java', - 'projectTgt\t:app\t/abs/app/build/classes', - 'root\tr1\t:app\truntimeClasspath\t1', + 'projectSrc\t:app\t/abs/app/src/main/java\tmain', + 'projectTgt\t:app\t/abs/app/build/classes\tmain', + 'root\tr1\t:app\truntimeClasspath\t1\tmain\truntime', 'node\tr1\tcom.example:lib:jar:1.0\tcom.example\tlib\t1.0\tjar\t\t1', 'node\tr1\tcom.example:bom:2.0\tcom.example\tbom\t2.0\t\t\t1', 'file\tr1\tcom.example:lib:jar:1.0\t/abs/lib.jar', @@ -98,9 +98,19 @@ describe('records → assemble → sidecar', () => { version: '1.0', subprojectDir: '/abs/app', dependencies: ['com.example:bom:2.0', 'com.example:lib:jar:1.0'], - targets: ['/abs/app/build/classes'], - sources: ['/abs/app/src/main/java'], - classpath: ['com.example:lib:jar:1.0', 'com.example:bom:2.0'], + outputs: { + main: { + sources: ['/abs/app/src/main/java'], + targets: ['/abs/app/build/classes'], + }, + }, + classpaths: { + main: [ + { project: ':app', outputs: 'main' }, + { id: 'com.example:lib:jar:1.0' }, + { id: 'com.example:bom:2.0' }, + ], + }, }, ]) @@ -119,12 +129,12 @@ describe('records → assemble → sidecar', () => { const records = [ 'meta\tgradle\t8.10\t17', 'project\t:app\tg\tapp\t1\tapp', - 'projectSrc\t:app\t/abs/app/src', - 'projectTgt\t:app\t/abs/app/classes', + 'projectSrc\t:app\t/abs/app/src\tmain', + 'projectTgt\t:app\t/abs/app/classes\tmain', 'excludedProject\t:lib', - 'projectSrc\t:lib\t/abs/lib/src', - 'projectTgt\t:lib\t/abs/lib/classes', - 'root\t0\t:app\tcompileClasspath\t1', + 'projectSrc\t:lib\t/abs/lib/src\tmain', + 'projectTgt\t:lib\t/abs/lib/classes\tmain', + 'root\t0\t:app\truntimeClasspath\t1\tmain\truntime', 'node\t0\t:lib\tg\tlib\t1\tjar\t\t1\t:lib', ].join('\n') const { artifactPaths, facts } = assembleFacts(parseRecords(records), { @@ -132,11 +142,15 @@ describe('records → assemble → sidecar', () => { }) expect(facts.projects?.map(p => p.id)).toEqual([':app']) - expect(artifactPaths.pathsById.get(':lib')).toEqual({ + expect(artifactPaths.outputsByProject.get(':lib')?.get('main')).toEqual({ sources: ['/abs/lib/src'], targets: ['/abs/lib/classes'], }) - expect([...artifactPaths.classpathByProject.keys()]).toEqual([':app']) + expect([...artifactPaths.classpathsByProject.keys()]).toEqual([':app']) + expect(artifactPaths.classpathsByProject.get(':app')?.get('main')).toEqual([ + { project: ':app', outputs: 'main' }, + { project: ':lib', outputs: 'main' }, + ]) }) it('states no targets for an artifact that failed to resolve or is gone from disk', () => { const records = [ @@ -220,15 +234,16 @@ describe('records → assemble → sidecar', () => { 'meta\tgradle\t8.0\t17', 'project\t:a\tcom.example\ta\t1.0\ta', 'project\t:b\tcom.example\tb\t1.0\tb', - 'root\tr1\t:a\truntimeClasspath\t1', + 'root\tr1\t:a\truntimeClasspath\t1\tmain\truntime', 'node\tr1\tg:lib:jar:1\tg\tlib\t1\tjar\t\t1', 'node\tr1\tg:dep:jar:1\tg\tdep\t1\tjar\t\t0', 'edge\tr1\tg:lib:jar:1\tg:dep:jar:1', - 'root\tr2\t:b\truntimeClasspath\t1', + 'root\tr2\t:b\truntimeClasspath\t1\tmain\truntime', 'node\tr2\tg:lib:jar:1\tg\tlib\t1\tjar\t\t1', 'node\tr2\tg:dep:jar:2\tg\tdep\t2\tjar\t\t0', 'edge\tr2\tg:lib:jar:1\tg:dep:jar:2', - 'root\tr3\t:b\ttestRuntimeClasspath\t0', + 'outputSet\t:b\ttest\tmain', + 'root\tr3\t:b\ttestRuntimeClasspath\t0\ttest\truntime', 'node\tr3\tg:junit:jar:4\tg\tjunit\t4\tjar\t\t1', ].join('\n') const { artifactPaths, facts } = assembleFacts(parseRecords(records), { @@ -262,12 +277,25 @@ describe('records → assemble → sidecar', () => { p, ]), ) - expect(byName.get('a')?.classpath).toEqual(['g:lib:jar:1', 'g:dep:jar:1']) - expect(byName.get('b')?.classpath).toEqual([ - 'g:lib:jar:1', - 'g:dep:jar:2', - 'g:junit:jar:4', - ]) + expect(byName.get('a')?.classpaths).toEqual({ + main: [ + { project: ':a', outputs: 'main' }, + { id: 'g:lib:jar:1' }, + { id: 'g:dep:jar:1' }, + ], + }) + expect(byName.get('b')?.classpaths).toEqual({ + main: [ + { project: ':b', outputs: 'main' }, + { id: 'g:lib:jar:1' }, + { id: 'g:dep:jar:2' }, + ], + test: [ + { project: ':b', outputs: 'test' }, + { project: ':b', outputs: 'main' }, + { id: 'g:junit:jar:4' }, + ], + }) expect(byName.get('b')?.dependencies).toEqual([ 'g:junit:jar:4', 'g:lib:jar:1', @@ -417,17 +445,18 @@ describe('records → assemble → sidecar', () => { const records = [ 'meta\tgradle\t8.0\t17', 'project\t:a:util\tex\tutil\t1\ta/util', - 'projectSrc\t:a:util\t/abs/a/util/src', + 'projectSrc\t:a:util\t/abs/a/util/src\tmain', 'project\t:b:util\tex\tutil\t1\tb/util', - 'projectSrc\t:b:util\t/abs/b/util/src', + 'projectSrc\t:b:util\t/abs/b/util/src\tmain', + 'projectSrc\t:b:util\t/abs/b/util/fixtures\ttestFixtures', 'project\t:app\tex\tapp\t1\tapp', - 'root\tr1\t:app\truntimeClasspath\t1', - 'node\tr1\tex:util:jar:1\tex\tutil\t1\tjar\t\t1\t:b:util', - 'node\tr1\tex:util:jar:test-fixtures:1\tex\tutil\t1\tjar\ttest-fixtures\t1\t:b:util', + 'root\tr1\t:app\truntimeClasspath\t1\tmain\truntime', + 'node\tr1\tex:util:jar:1\tex\tutil\t1\tjar\t\t1\t:b:util\tmain', + 'node\tr1\tex:util:jar:test-fixtures:1\tex\tutil\t1\tjar\ttest-fixtures\t1\t:b:util\ttestFixtures', 'node\tr1\tg:ext:jar:2\tg\text\t2\tjar\t\t0\t', 'edge\tr1\tex:util:jar:test-fixtures:1\tex:util:jar:1', 'edge\tr1\tex:util:jar:1\tg:ext:jar:2', - 'root\tr2\t:b:util\truntimeClasspath\t1', + 'root\tr2\t:b:util\truntimeClasspath\t1\tmain\truntime', 'node\tr2\tex:util:jar:1\tex\tutil\t1\tjar\t\t1\t:a:util', ].join('\n') const { artifactPaths, facts } = assembleFacts(parseRecords(records), { @@ -444,17 +473,24 @@ describe('records → assemble → sidecar', () => { ':app': [':b:util(g:ext:jar:2)'], ':b:util': [':a:util'], }) - expect(artifactPaths.classpathByProject.get(':app')).toEqual([ - ':b:util', - 'g:ext:jar:2', + expect(artifactPaths.classpathsByProject.get(':app')?.get('main')).toEqual([ + { project: ':app', outputs: 'main' }, + { project: ':b:util', outputs: 'main' }, + { project: ':b:util', outputs: 'testFixtures' }, + { id: 'g:ext:jar:2' }, ]) - expect(artifactPaths.classpathByProject.get(':b:util')).toEqual([':a:util']) - expect(artifactPaths.pathsById.get(':a:util')?.sources).toEqual([ - '/abs/a/util/src', - ]) - expect(artifactPaths.pathsById.get(':b:util')?.sources).toEqual([ - '/abs/b/util/src', + expect( + artifactPaths.classpathsByProject.get(':b:util')?.get('main'), + ).toEqual([ + { project: ':b:util', outputs: 'main' }, + { project: ':a:util', outputs: 'main' }, ]) + expect( + Object.fromEntries(artifactPaths.outputsByProject.get(':b:util') ?? []), + ).toEqual({ + main: { sources: ['/abs/b/util/src'], targets: [] }, + testFixtures: { sources: ['/abs/b/util/fixtures'], targets: [] }, + }) }) it('marks each subproject with its own build files and no component with any', () => { @@ -544,141 +580,131 @@ describe('records → assemble → sidecar', () => { }) }) -function classpath(records: string[], project: string): string[] | undefined { - return assembleFacts(parseRecords(records.join('\n')), { - fileExists: () => true, - }).artifactPaths.classpathByProject.get(project) +function classpaths( + records: string[], + project: string, +): Record { + return Object.fromEntries( + assembleFacts(parseRecords(records.join('\n')), { + fileExists: () => true, + }).artifactPaths.classpathsByProject.get(project) ?? [], + ) } -describe('classpath', () => { - it("follows each root's order, runtime before compile-only, test and tooling roots", () => { +describe('classpaths', () => { + it("gives each output set its runtime classpath in the root's order, then compile-only entries", () => { expect( - classpath( + classpaths( [ 'meta\tgradle\t8.10\t17', 'project\t:app\tg\tapp\t1\tapp', + 'outputSet\t:app\tmain\t', + 'outputSet\t:app\ttest\tmain', + 'outputSet\t:app\ttestFixtures\tmain', 'root\t0\t:app\tannotationProcessor\t0', 'node\t0\tg:proc:jar:1\tg\tproc\t1\tjar\t\t1', - 'root\t1\t:app\tcompileClasspath\t1', + 'root\t1\t:app\tcompileClasspath\t1\tmain\tcompile', 'node\t1\tg:api:jar:1\tg\tapi\t1\tjar\t\t1', 'node\t1\tg:servlet:jar:3\tg\tservlet\t3\tjar\t\t1', - 'root\t2\t:app\truntimeClasspath\t1', + 'root\t2\t:app\truntimeClasspath\t1\tmain\truntime', 'node\t2\tg:z:jar:1\tg\tz\t1\tjar\t\t1', - 'node\t2\tg:api:jar:1\tg\tapi\t1\tjar\t\t1', - 'root\t3\t:app\ttestRuntimeClasspath\t0', + 'node\t2\tg:api:jar:2\tg\tapi\t2\tjar\t\t1', + 'root\t3\t:app\ttestRuntimeClasspath\t0\ttest\truntime', 'node\t3\tg:junit:jar:4\tg\tjunit\t4\tjar\t\t1', - 'node\t3\tg:z:jar:1\tg\tz\t1\tjar\t\t1', + 'node\t3\tg:z:jar:2\tg\tz\t2\tjar\t\t1', ], ':app', ), - ).toEqual([ - 'g:z:jar:1', - 'g:api:jar:1', - 'g:servlet:jar:3', - 'g:junit:jar:4', - 'g:proc:jar:1', - ]) - }) - - it('keeps the version of a module the highest-ranked root resolved, with its other artifacts', () => { - expect( - classpath( - [ - 'meta\tgradle\t8.10\t17', - 'project\t:app\tg\tapp\t1\tapp', - 'root\t0\t:app\ttestRuntimeClasspath\t0', - 'node\t0\tg:guava:jar:32\tg\tguava\t32\tjar\t\t1', - 'node\t0\tg:netty:jar:linux:4.1\tg\tnetty\t4.1\tjar\tlinux\t1', - 'root\t1\t:app\truntimeClasspath\t1', - 'node\t1\tg:guava:jar:30\tg\tguava\t30\tjar\t\t1', - 'node\t1\tg:netty:jar:4.1\tg\tnetty\t4.1\tjar\t\t1', - ], - ':app', - ), - ).toEqual(['g:guava:jar:30', 'g:netty:jar:4.1', 'g:netty:jar:linux:4.1']) - }) - - it("ranks Gradle's legacy runtime and compile behind the classpath configurations", () => { - expect( - classpath( - [ - 'meta\tgradle\t6.9\t11', - 'project\t:app\tg\tapp\t1\tapp', - 'root\t0\t:app\tcompile\t1', - 'node\t0\tg:foo:jar:1\tg\tfoo\t1\tjar\t\t1', - 'root\t1\t:app\tcompileClasspath\t1', - 'node\t1\tg:bar:jar:1\tg\tbar\t1\tjar\t\t1', - 'root\t2\t:app\truntime\t1', - 'node\t2\tg:foo:jar:1\tg\tfoo\t1\tjar\t\t1', - 'root\t3\t:app\truntimeClasspath\t1', - 'node\t3\tg:bar:jar:1\tg\tbar\t1\tjar\t\t1', - 'node\t3\tg:foo:jar:2\tg\tfoo\t2\tjar\t\t0', - 'edge\t3\tg:bar:jar:1\tg:foo:jar:2', - ], - ':app', - ), - ).toEqual(['g:bar:jar:1', 'g:foo:jar:2']) + ).toEqual({ + main: [ + { project: ':app', outputs: 'main' }, + { id: 'g:z:jar:1' }, + { id: 'g:api:jar:2' }, + { id: 'g:servlet:jar:3' }, + ], + test: [ + { project: ':app', outputs: 'test' }, + { project: ':app', outputs: 'main' }, + { id: 'g:junit:jar:4' }, + { id: 'g:z:jar:2' }, + ], + testFixtures: [ + { project: ':app', outputs: 'testFixtures' }, + { project: ':app', outputs: 'main' }, + ], + }) }) - it('tells test configurations by name segment, sbt it among them', () => { + it("gives a set without a runtime classpath its own code, and adds no set it doesn't declare", () => { expect( - classpath( + classpaths( [ 'meta\tgradle\t8.10\t17', 'project\t:app\tg\tapp\t1\tapp', - 'root\t0\t:app\ttestRuntimeClasspath\t0', - 'node\t0\tg:x:jar:3\tg\tx\t3\tjar\t\t1', - 'root\t1\t:app\tlatestRuntimeClasspath\t1', - 'node\t1\tg:x:jar:2\tg\tx\t2\tjar\t\t1', + 'outputSet\t:app\tmain\t', + 'outputSet\t:app\tjvmTest\tjvm', + 'root\t0\t:app\tcompileClasspath\t1\tmain\tcompile', + 'node\t0\tg:api:jar:1\tg\tapi\t1\tjar\t\t1', ], ':app', ), - ).toEqual(['g:x:jar:2']) - expect( - classpath( - [ - 'meta\tsbt\t1.9.9\t17', - 'project\tapp\tg\tapp\t1\tapp', - 'root\t0\tapp\tscala-tool\t0', - 'node\t0\tg:x:jar:2\tg\tx\t2\tjar\t\t1', - 'root\t1\tapp\tit\t0', - 'node\t1\tg:x:jar:1\tg\tx\t1\tjar\t\t1', - ], - 'app', - ), - ).toEqual(['g:x:jar:1']) + ).toEqual({ + main: [{ project: ':app', outputs: 'main' }], + jvmTest: [ + { project: ':app', outputs: 'jvmTest' }, + { project: ':app', outputs: 'jvm' }, + ], + }) }) - it('takes a classpath the build tool states as it is, by component id', () => { + it('takes classpaths the build tool states as they are, by component id', () => { expect( - classpath( + classpaths( [ 'meta\tmaven\t3.9.6\t17', 'project\tg:app:1\tg\tapp\t1\tapp', 'project\tg:lib:1\tg\tlib\t1\tlib', + 'outputSet\tg:app:1\tmain\t', + 'outputSet\tg:app:1\ttest\tmain', 'root\t0\tg:app:1\tcompile\t1', 'node\t0\tg:z:jar:1\tg\tz\t1\tjar\t\t1', 'node\t0\tg:lib:1\tg\tlib\t1\t\t\t1\tg:lib:1', 'root\t1\tg:app:1\ttest\t0', 'node\t1\tg:bcpkix:jar:1\tg\tbcpkix\t1\tjar\t\t1', - 'classpath\tg:app:1\tg:bcpkix:jar:1', - 'classpath\tg:app:1\tg:lib:1', - 'classpath\tg:app:1\tg:z:jar:1', + 'classpath\tg:app:1\tmain\tg:lib:1\tmain', + 'classpath\tg:app:1\tmain\tg:z:jar:1\t', + 'classpath\tg:app:1\ttest\tg:bcpkix:jar:1\t', + 'classpath\tg:app:1\ttest\tg:lib:1\tmain,test', + 'classpath\tg:app:1\ttest\tg:z:jar:1\t', ], 'g:app:1', ), - ).toEqual(['g:bcpkix:jar:1', 'g:lib:1', 'g:z:jar:1']) + ).toEqual({ + main: [ + { project: 'g:app:1', outputs: 'main' }, + { project: 'g:lib:1', outputs: 'main' }, + { id: 'g:z:jar:1' }, + ], + test: [ + { project: 'g:app:1', outputs: 'test' }, + { project: 'g:app:1', outputs: 'main' }, + { id: 'g:bcpkix:jar:1' }, + { project: 'g:lib:1', outputs: 'main' }, + { project: 'g:lib:1', outputs: 'test' }, + { id: 'g:z:jar:1' }, + ], + }) }) it('rejects a stated classpath entry that no root of the project resolved', () => { expect(() => - classpath( + classpaths( [ 'meta\tmaven\t3.9.6\t17', 'project\tg:app:1\tg\tapp\t1\tapp', 'root\t0\tg:app:1\tcompile\t1', 'node\t0\tg:z:jar:1\tg\tz\t1\tjar\t\t1', - 'classpath\tg:app:1\tg:y:jar:1', + 'classpath\tg:app:1\tmain\tg:y:jar:1\t', ], 'g:app:1', ), diff --git a/src/commands/manifest/scripts/facts.mts b/src/commands/manifest/scripts/facts.mts index 97f8cfb786..f74ba0d403 100644 --- a/src/commands/manifest/scripts/facts.mts +++ b/src/commands/manifest/scripts/facts.mts @@ -58,14 +58,27 @@ export type SocketFactsSbomProject = AnyPURL & { children: number[] } -// Resolved on-disk paths for a --with-files run, keyed by component or project -// id (a project and its own component share one). `targets` = classpath -// entries (jars / module output dirs), absent when the component's artifact -// did not resolve; `sources` = module source roots. +export type OutputPaths = { + sources: string[] + targets: string[] +} + +// One entry of a classpath, in the order the JVM searches it. +export type ClasspathEntry = + // An external artifact, by component id. + | { id: string } + // One output set (`main`, `test`, `testFixtures`, ...) of a build project. + | { project: string; outputs: string } + +// Resolved on-disk paths for a --with-files run, and each project's classpaths. export type ResolvedArtifactPaths = { - pathsById: Map - // Component ids on each project's classpath, in order, keyed by project id. - classpathByProject: Map + // External artifacts by component id: `targets` are their files, absent when + // the artifact did not resolve. + pathsById: Map + // Build projects' output sets by name, keyed by project id. + outputsByProject: Map> + // Each project's classpaths by output set, keyed by project id. + classpathsByProject: Map> // Component ids of each project's direct dependencies, keyed by project id. directDependenciesByProject: Map } diff --git a/src/commands/manifest/scripts/maven-extension/src/main/java/tech/coana/socket/SocketFactsRecordsEngine.java b/src/commands/manifest/scripts/maven-extension/src/main/java/tech/coana/socket/SocketFactsRecordsEngine.java index 45eb890739..b297c49c0c 100644 --- a/src/commands/manifest/scripts/maven-extension/src/main/java/tech/coana/socket/SocketFactsRecordsEngine.java +++ b/src/commands/manifest/scripts/maven-extension/src/main/java/tech/coana/socket/SocketFactsRecordsEngine.java @@ -144,9 +144,18 @@ public void run(MavenSession session, List reactor, File rootDir, File pom = module.getFile(); if (pom != null && pom.isFile()) rec(lines, "projectBuild", key, SocketSupport.relativePath(rootDir.toPath(), pom.toPath())); } + // Maven puts a module's main classes on its test classpath. + if (!excluded || opts.withFiles) { + rec(lines, "outputSet", key, "main", ""); + rec(lines, "outputSet", key, "test", "main"); + } if (opts.withFiles) { - for (String s : collectSources(module)) rec(lines, "projectSrc", key, s); - for (String t : collectTargets(module)) rec(lines, "projectTgt", key, t); + for (String set : new String[] {"main", "test"}) { + boolean test = "test".equals(set); + for (String s : collectSources(module, test)) rec(lines, "projectSrc", key, s, set); + String target = test ? module.getBuild().getTestOutputDirectory() : module.getBuild().getOutputDirectory(); + if (target != null) rec(lines, "projectTgt", key, new File(target).getAbsolutePath(), set); + } } } @@ -431,7 +440,7 @@ private String visit( ? upsert(nodes, id, artifact.getGroupId(), artifact.getArtifactId(), "", "", version, gav) : upsert(nodes, id, artifact.getGroupId(), artifact.getArtifactId(), type == null ? "" : type, classifier == null ? "" : classifier, version, ""); - ("test".equals(scope) ? order.test : order.main).add(id); + order.add(id, scope, internal ? ("test-jar".equals(type) || "tests".equals(classifier) ? "test" : "main") : null); // Maven wrote each accepted node's resolved file back onto the node; a reactor module reports its // own dirs through its `project` record instead of a `file` record. if (!internal && opts.withFiles) { @@ -508,13 +517,16 @@ private int emitModuleRoots( return rootIdx; } - // Maven builds every classpath of a module from one graph, in the pre-order its own - // RepositoryUtils.toArtifacts walks, filtered by each artifact's effective scope. The test scope goes - // last, so that a test-only artifact never shadows one main code runs with. + // Maven's pre-order walk of the module's graph (RepositoryUtils.toArtifacts), filtered by scope; `main` + // is the runtime classpath, then the compile-only (provided, system) entries. private static void emitClasspath(List lines, String projectKey, ClasspathOrder order) { - for (String id : order.main) rec(lines, "classpath", projectKey, id); - for (String id : order.test) { - if (!order.main.contains(id)) rec(lines, "classpath", projectKey, id); + emitClasspath(lines, projectKey, "main", order.main()); + emitClasspath(lines, projectKey, "test", order.test); + } + + private static void emitClasspath(List lines, String projectKey, String name, Map> entries) { + for (Map.Entry> e : entries.entrySet()) { + rec(lines, "classpath", projectKey, name, e.getKey(), String.join(",", e.getValue())); } } @@ -592,29 +604,19 @@ private Set readPopulateGavs(Options opts) throws IOException { // Configured source roots, emitted unconditionally (like gradle's srcDirs / sbt's sourceDirectories): // the analysis never builds the project, so these need not exist yet. - private List collectSources(MavenProject module) { + private List collectSources(MavenProject module, boolean test) { Set sources = new TreeSet<>(); - addPaths(sources, module.getCompileSourceRoots()); - addPaths(sources, module.getTestCompileSourceRoots()); - for (Resource r : module.getBuild().getResources()) addPath(sources, r.getDirectory()); - for (Resource r : module.getBuild().getTestResources()) addPath(sources, r.getDirectory()); + addPaths(sources, test ? module.getTestCompileSourceRoots() : module.getCompileSourceRoots()); + for (Resource r : test ? module.getBuild().getTestResources() : module.getBuild().getResources()) { + addPath(sources, r.getDirectory()); + } // generated-source roots aren't on the model without a build; best-effort pick up the // conventional dirs only if a prior `mvn compile` already produced them (else they'd be guesses). String buildDir = module.getBuild().getDirectory(); - addExisting(sources, buildDir + File.separator + "generated-sources"); - addExisting(sources, buildDir + File.separator + "generated-test-sources"); + addExisting(sources, buildDir + File.separator + (test ? "generated-test-sources" : "generated-sources")); return new ArrayList<>(sources); } - // Configured compiled-output dirs, emitted unconditionally (like gradle's classesDirs / sbt's - // classDirectory): the analysis never builds the project, so these need not exist yet. - private List collectTargets(MavenProject module) { - Set targets = new TreeSet<>(); - addPath(targets, module.getBuild().getOutputDirectory()); - addPath(targets, module.getBuild().getTestOutputDirectory()); - return new ArrayList<>(targets); - } - private static void addPaths(Set acc, List dirs) { if (dirs == null) return; for (String d : dirs) addPath(acc, d); @@ -888,11 +890,35 @@ public int hashCode() { } } - // Node ids in walk order, by whether the artifact reached them with the test scope. A sibling module's - // variants share a node, and can reach it with different scopes. + // Node ids in walk order by classpath, each with the output sets of a reactor module its artifacts bring in + // (its jar main, its test-jar test). private static final class ClasspathOrder { - final Set main = new LinkedHashSet<>(); - final Set test = new LinkedHashSet<>(); + final Map> runtime = new LinkedHashMap<>(); + final Map> compileOnly = new LinkedHashMap<>(); + final Map> test = new LinkedHashMap<>(); + + void add(String id, String scope, String outputSet) { + if ("compile".equals(scope) || "runtime".equals(scope)) put(runtime, id, outputSet); + else if (!"test".equals(scope)) put(compileOnly, id, outputSet); + put(test, id, outputSet); + } + + // The runtime classpath, then the compile-only entries. + Map> main() { + Map> main = new LinkedHashMap<>(); + for (Map> part : Arrays.asList(runtime, compileOnly)) { + part.forEach((id, sets) -> { + put(main, id, null); + main.get(id).addAll(sets); + }); + } + return main; + } + + private static void put(Map> entries, String id, String outputSet) { + Set sets = entries.computeIfAbsent(id, k -> new LinkedHashSet<>()); + if (outputSet != null) sets.add(outputSet); + } } private static final class Node { diff --git a/src/commands/manifest/scripts/records.mts b/src/commands/manifest/scripts/records.mts index 5089d5fcb3..69c8d66eb9 100644 --- a/src/commands/manifest/scripts/records.mts +++ b/src/commands/manifest/scripts/records.mts @@ -1,3 +1,4 @@ +import type { OutputPaths } from './facts.mts' import type { ResolutionFailure, Unscannable } from './resolution-report.mts' // Line-protocol the build-tool scripts emit to a records file (NOT stdout — sbt @@ -10,14 +11,15 @@ import type { ResolutionFailure, Unscannable } from './resolution-report.mts' // buildRoot path (absolute; the facts file's directory) // entry path (file-addressed builds; build-root-relative) // project projectKey group name version dir -// projectSrc projectKey path (--with-files only) -// projectTgt projectKey path (--with-files only) +// projectSrc projectKey path outputSet (--with-files only) +// projectTgt projectKey path outputSet (--with-files only) +// outputSet projectKey outputSet includes (the project's own sets on its classpath) // projectBuild projectKey path (build-root-relative; may be absent on disk) // excludedProject projectKey (--with-files only; paths without a project) -// root rootId projectKey config prod(0|1) -// node rootId coordId group name version ext classifier direct(0|1) project +// root rootId projectKey config prod(0|1) outputSet role(runtime|compile) +// node rootId coordId group name version ext classifier direct(0|1) project outputSets(brought in) // edge rootId parentCoordId childCoordId -// classpath projectKey coordId (in order; see below) +// classpath projectKey outputSet coordId outputSets (in order; see below) // file rootId coordId path (--with-files only) // unfetched rootId coordId (--with-files only; requested artifact not fetched) // scanned config @@ -31,8 +33,9 @@ import type { ResolutionFailure, Unscannable } from './resolution-report.mts' // coordinate key (`group:name:ext:classifier:version`, empty segments dropped), // used opaquely as the per-root node key. Unknown tags are ignored. // -// `node` records follow the root's classpath order. `classpath` records state a -// project's whole classpath (Maven); without them the assembler derives it. +// Output sets are a project's code by source set (`main`, `test`, ...). A root +// with one is that set's runtime or compile classpath, its nodes in classpath +// order; `classpath` records state Maven's outright. export type RawCoord = { group: string @@ -49,6 +52,8 @@ export type RawNode = { // projectKey of the build project this node resolves to; empty for an // external artifact. project: string + // The build project's output sets this dependency brings in. + outputSets: string[] // --with-files only. targets: string[] unfetched: boolean @@ -57,8 +62,10 @@ export type RawNode = { export type RawRoot = { rootId: string projectKey: string - config: string prod: boolean + // The output set whose classpath this root is, or '' for none. + outputSet: string + role: 'runtime' | 'compile' | '' nodes: Map edges: Array<[string, string]> } @@ -69,12 +76,18 @@ export type RawProject = { name: string version: string dir: string - sources: string[] - targets: string[] + outputs: Map + // Each output set, with the project's other output sets on its classpath. + outputSets: Map // The project's build files, build-root-relative. buildFiles: string[] } +export type StatedClasspathEntry = { + coordId: string + outputSets: string[] +} + export type ParsedRecords = { tool: string toolVersion: string @@ -89,8 +102,8 @@ export type ParsedRecords = { // the kept projects depending on them. excludedProjects: Map roots: Map - // Keyed by projectKey; coordIds of the project's roots' nodes, in order. - classpaths: Map + // By projectKey, then output set: nodes of the project's roots, in order. + classpaths: Map> scannedConfigs: string[] failures: ResolutionFailure[] unscannable: Unscannable[] @@ -117,6 +130,14 @@ function bool(s: string | undefined): boolean { return s === '1' || s === 'true' } +function list(s: string | undefined): string[] { + return s ? s.split(',') : [] +} + +function role(s: string | undefined): RawRoot['role'] { + return s === 'runtime' || s === 'compile' ? s : '' +} + export function parseRecords(text: string): ParsedRecords { const result: ParsedRecords = { tool: '', @@ -141,8 +162,9 @@ export function parseRecords(text: string): ParsedRecords { r = { rootId: id, projectKey: '', - config: '', prod: false, + outputSet: '', + role: '', nodes: new Map(), edges: [], } @@ -159,8 +181,8 @@ export function parseRecords(text: string): ParsedRecords { name: '', version: '', dir: '', - sources: [], - targets: [], + outputs: new Map(), + outputSets: new Map(), buildFiles: [], } result.projects.set(key, p) @@ -168,6 +190,16 @@ export function parseRecords(text: string): ParsedRecords { return p } + const outputs = (key: string, set: string): OutputPaths => { + const all = project(key).outputs + let o = all.get(set) + if (!o) { + o = { sources: [], targets: [] } + all.set(set, o) + } + return o + } + for (const rawLine of text.split('\n')) { if (!rawLine) { continue @@ -194,13 +226,18 @@ export function parseRecords(text: string): ParsedRecords { break } case 'projectSrc': - if (f[2]) { - project(f[1] ?? '').sources.push(f[2]) + if (f[2] && f[3]) { + outputs(f[1] ?? '', f[3]).sources.push(f[2]) } break case 'projectTgt': + if (f[2] && f[3]) { + outputs(f[1] ?? '', f[3]).targets.push(f[2]) + } + break + case 'outputSet': if (f[2]) { - project(f[1] ?? '').targets.push(f[2]) + project(f[1] ?? '').outputSets.set(f[2], list(f[3])) } break case 'excludedProject': @@ -214,8 +251,9 @@ export function parseRecords(text: string): ParsedRecords { case 'root': { const r = root(f[1] ?? '') r.projectKey = f[2] ?? '' - r.config = f[3] ?? '' r.prod = bool(f[4]) + r.outputSet = f[5] ?? '' + r.role = role(f[6]) break } case 'node': { @@ -232,6 +270,7 @@ export function parseRecords(text: string): ParsedRecords { }, direct: bool(f[8]), project: f[9] ?? '', + outputSets: list(f[10]), targets: [], unfetched: false, }) @@ -247,13 +286,19 @@ export function parseRecords(text: string): ParsedRecords { } case 'classpath': { const key = f[1] ?? '' - let cp = result.classpaths.get(key) + let byName = result.classpaths.get(key) + if (!byName) { + byName = new Map() + result.classpaths.set(key, byName) + } + const name = f[2] ?? '' + let cp = byName.get(name) if (!cp) { cp = [] - result.classpaths.set(key, cp) + byName.set(name, cp) } - if (f[2]) { - cp.push(f[2]) + if (f[3]) { + cp.push({ coordId: f[3], outputSets: list(f[4]) }) } break } diff --git a/src/commands/manifest/scripts/sidecar.mts b/src/commands/manifest/scripts/sidecar.mts index 136934db90..e98a048150 100644 --- a/src/commands/manifest/scripts/sidecar.mts +++ b/src/commands/manifest/scripts/sidecar.mts @@ -1,30 +1,32 @@ import type { AnyPURL, + ClasspathEntry, + OutputPaths, ResolvedArtifactPaths, SocketFactsSbom, SocketFactsSbomComponent, SocketFactsSbomProject, } from './facts.mts' +// A build project's code by output set (`main`, `test`, `testFixtures`, ...). +export type SidecarOutputs = Record + export type SidecarComponentEntry = SocketFactsSbomComponent & { - // Classpath entries (jars, or a sibling project's own build output dirs - // when this component is that project). `[]` means resolved and found - // nothing (e.g. a pom/BOM); undefined means paths were not resolved. + // An external artifact's files: `[]` resolved and found nothing (a pom/BOM), + // undefined not resolved. `sources` is always `[]`. targets?: string[] | undefined - // First-party source roots; `[]` for a genuinely external dependency (still - // attempted, nothing to find), not undefined. sources?: string[] | undefined + // A sibling build project's output sets, in place of `targets`/`sources`. + outputs?: SidecarOutputs | undefined } export type SidecarProjectEntry = Omit & { - targets?: string[] | undefined - sources?: string[] | undefined + outputs?: SidecarOutputs | undefined // Ids of this facts file's components[] the project depends on directly. dependencies: string[] - // Ids of this facts file's components[] on the project's classpath: one - // version per module, main code's entries before test-only and tooling ones. + // The runtime classpath of each output set, the project's own code first. // When artifact paths were resolved, in the order the JVM searches them. - classpath: string[] + classpaths: Record } // Frozen contract with `coana run --compute-artifacts-sidecar`; change only @@ -43,8 +45,8 @@ export type ResolvedPathsSidecar = Record< projects: SidecarProjectEntry[] // This reactor's dependency-position entries: genuinely external // artifacts, and dependency edges that resolve to a sibling first-party - // project (reported via that project's own source/target roots instead - // of a jar path). + // project (reported via that project's output sets instead of a jar + // path). components: SidecarComponentEntry[] } > @@ -54,18 +56,26 @@ export type SidecarAccumulator = Map< { projects: SidecarProjectEntry[]; components: SidecarComponentEntry[] } > -// `[]` means resolved and found nothing (e.g. a pom/BOM with no artifact); no -// `targets` means the artifact did not resolve. +// A build project carries its output sets; an external artifact its files, +// `targets` absent when it did not resolve. function attachPaths( entry: T, artifactPaths: ResolvedArtifactPaths, -): T & { targets?: string[]; sources: string[] } { +): T & { + outputs?: SidecarOutputs + sources?: string[] + targets?: string[] +} { + const outputs = artifactPaths.outputsByProject.get(entry.id) + if (outputs) { + return { ...entry, outputs: Object.fromEntries(outputs) } + } const paths = artifactPaths.pathsById.get(entry.id) const targets = paths ? paths.targets : [] return { ...entry, ...(targets ? { targets: [...targets] } : {}), - sources: [...(paths?.sources ?? [])], + sources: [], } } @@ -104,7 +114,9 @@ export function accumulateSidecar( dependencies: [ ...(artifactPaths.directDependenciesByProject.get(proj.id) ?? []), ], - classpath: [...(artifactPaths.classpathByProject.get(proj.id) ?? [])], + classpaths: Object.fromEntries( + artifactPaths.classpathsByProject.get(proj.id) ?? [], + ), }), ), }) diff --git a/src/commands/manifest/scripts/sidecar.test.mts b/src/commands/manifest/scripts/sidecar.test.mts index 412819d19b..53cedb7730 100644 --- a/src/commands/manifest/scripts/sidecar.test.mts +++ b/src/commands/manifest/scripts/sidecar.test.mts @@ -14,7 +14,8 @@ import type { SidecarAccumulator } from './sidecar.mts' function emptyArtifactPaths(): ResolvedArtifactPaths { return { pathsById: new Map(), - classpathByProject: new Map(), + outputsByProject: new Map(), + classpathsByProject: new Map(), directDependenciesByProject: new Map(), } } @@ -24,7 +25,7 @@ function mkComponentFixture(target: string): { paths: ResolvedArtifactPaths } { const paths = emptyArtifactPaths() - paths.pathsById.set('g:a:jar:1', { sources: [], targets: [target] }) + paths.pathsById.set('g:a:jar:1', { targets: [target] }) return { facts: { components: [ @@ -67,7 +68,10 @@ describe('compute-artifacts sidecar', () => { ], } const artifactPaths = emptyArtifactPaths() - artifactPaths.classpathByProject.set(':app', ['g:a:jar:1']) + artifactPaths.classpathsByProject.set( + ':app', + new Map([['main', [{ id: 'g:a:jar:1' }]]]), + ) artifactPaths.directDependenciesByProject.set(':app', ['g:a:jar:1']) const acc: SidecarAccumulator = new Map() @@ -87,12 +91,12 @@ describe('compute-artifacts sidecar', () => { name: 'app', subprojectDir: 'app', dependencies: ['g:a:jar:1'], - classpath: ['g:a:jar:1'], + classpaths: { main: [{ id: 'g:a:jar:1' }] }, }) expect(entry.components[0]).toEqual(facts.components[0]) }) - it('carries a component through with resolved targets/sources attached, keyed by its own facts file', () => { + it('carries a component through with resolved targets attached, keyed by its own facts file', () => { const facts: SocketFactsSbom = { components: [ { @@ -107,7 +111,6 @@ describe('compute-artifacts sidecar', () => { } const artifactPaths = emptyArtifactPaths() artifactPaths.pathsById.set('com.example:lib:jar:da517db', { - sources: ['/abs/lib/src/main/java'], targets: ['/abs/lib.jar'], }) @@ -127,7 +130,7 @@ describe('compute-artifacts sidecar', () => { qualifiers: { ext: 'jar' }, id: 'com.example:lib:jar:da517db', targets: ['/abs/lib.jar'], - sources: ['/abs/lib/src/main/java'], + sources: [], }, ], }, @@ -161,7 +164,7 @@ describe('compute-artifacts sidecar', () => { expect(entry.sources).toEqual([]) }) - it("gives each sibling project's component that project's own paths, even when they share a coordinate", () => { + it("gives each sibling project's component that project's own output sets, even when they share a coordinate", () => { const util = { type: 'maven', namespace: 'ex', @@ -180,14 +183,24 @@ describe('compute-artifacts sidecar', () => { ], } const artifactPaths = emptyArtifactPaths() - artifactPaths.pathsById.set(':a:util', { - sources: ['/abs/a/util/src'], - targets: ['/abs/a/util/classes'], - }) - artifactPaths.pathsById.set(':b:util', { - sources: ['/abs/b/util/src'], - targets: ['/abs/b/util/classes'], - }) + artifactPaths.outputsByProject.set( + ':a:util', + new Map([ + [ + 'main', + { sources: ['/abs/a/util/src'], targets: ['/abs/a/util/classes'] }, + ], + ]), + ) + artifactPaths.outputsByProject.set( + ':b:util', + new Map([ + [ + 'main', + { sources: ['/abs/b/util/src'], targets: ['/abs/b/util/classes'] }, + ], + ]), + ) const acc: SidecarAccumulator = new Map() accumulateSidecar( @@ -199,7 +212,11 @@ describe('compute-artifacts sidecar', () => { const entry = serializeSidecar(acc)['/root/gradle.socket.facts.json']! for (const entries of [entry.components, entry.projects]) { - expect(Object.fromEntries(entries.map(e => [e.id, e.sources]))).toEqual({ + expect( + Object.fromEntries( + entries.map(e => [e.id, e.outputs?.['main']?.sources]), + ), + ).toEqual({ ':a:util': ['/abs/a/util/src'], ':b:util': ['/abs/b/util/src'], }) @@ -234,7 +251,7 @@ describe('compute-artifacts sidecar', () => { expect(entry.direct).toBe(true) }) - it('carries a first-party module (project, not a component) source/target roots, keyed by its own facts file', () => { + it('carries a first-party module (project, not a component) output sets, keyed by its own facts file', () => { const facts: SocketFactsSbom = { // The app module is a project but nothing depends on it, so it is absent // from components — its source roots must still reach the sidecar. @@ -252,10 +269,25 @@ describe('compute-artifacts sidecar', () => { ], } const artifactPaths = emptyArtifactPaths() - artifactPaths.pathsById.set('com.example:app:1.0', { - sources: ['/abs/app/src/main/java'], - targets: ['/abs/app/build/classes'], - }) + artifactPaths.outputsByProject.set( + 'com.example:app:1.0', + new Map([ + [ + 'main', + { + sources: ['/abs/app/src/main/java'], + targets: ['/abs/app/classes'], + }, + ], + [ + 'test', + { + sources: ['/abs/app/src/test/java'], + targets: ['/abs/app/test-classes'], + }, + ], + ]), + ) const acc: SidecarAccumulator = new Map() accumulateSidecar(acc, facts, artifactPaths, '/root/app/.socket.facts.json') @@ -271,14 +303,22 @@ describe('compute-artifacts sidecar', () => { version: '1.0', subprojectDir: 'app', dependencies: [], - targets: ['/abs/app/build/classes'], - sources: ['/abs/app/src/main/java'], - classpath: [], + outputs: { + main: { + sources: ['/abs/app/src/main/java'], + targets: ['/abs/app/classes'], + }, + test: { + sources: ['/abs/app/src/test/java'], + targets: ['/abs/app/test-classes'], + }, + }, + classpaths: {}, }, ]) }) - it('attaches each project its own classpath ids, keyed by project id even within one directory', () => { + it('attaches each project its own classpaths, keyed by project id even within one directory', () => { const project = { type: 'maven', namespace: 'com.example', @@ -294,16 +334,22 @@ describe('compute-artifacts sidecar', () => { ], } const artifactPaths = emptyArtifactPaths() - artifactPaths.classpathByProject.set('x/a.xml', ['g:x:jar:1']) - artifactPaths.classpathByProject.set('x/b.xml', ['g:x:jar:2']) + artifactPaths.classpathsByProject.set( + 'x/a.xml', + new Map([['main', [{ id: 'g:x:jar:1' }]]]), + ) + artifactPaths.classpathsByProject.set( + 'x/b.xml', + new Map([['main', [{ id: 'g:x:jar:2' }]]]), + ) const acc: SidecarAccumulator = new Map() accumulateSidecar(acc, facts, artifactPaths, '/root/.socket.facts.json') const projects = serializeSidecar(acc)['/root/.socket.facts.json']!.projects - expect(projects.map(p => p.classpath)).toEqual([ - ['g:x:jar:1'], - ['g:x:jar:2'], + expect(projects.map(p => p.classpaths)).toEqual([ + { main: [{ id: 'g:x:jar:1' }] }, + { main: [{ id: 'g:x:jar:2' }] }, ]) }) @@ -339,15 +385,15 @@ describe('compute-artifacts sidecar', () => { ], } const pathsA = emptyArtifactPaths() - pathsA.pathsById.set('com.example:shared:1.0', { - sources: ['/root-a/src/main/java'], - targets: [], - }) + pathsA.outputsByProject.set( + 'com.example:shared:1.0', + new Map([['main', { sources: ['/root-a/src/main/java'], targets: [] }]]), + ) const pathsB = emptyArtifactPaths() - pathsB.pathsById.set('com.example:shared:1.0', { - sources: ['/root-b/src/main/java'], - targets: [], - }) + pathsB.outputsByProject.set( + 'com.example:shared:1.0', + new Map([['main', { sources: ['/root-b/src/main/java'], targets: [] }]]), + ) const acc: SidecarAccumulator = new Map() accumulateSidecar( @@ -369,10 +415,12 @@ describe('compute-artifacts sidecar', () => { '/root-b/.socket.facts.json', ]) expect( - resolved['/root-a/.socket.facts.json']!.projects[0]!.sources, + resolved['/root-a/.socket.facts.json']!.projects[0]!.outputs?.['main'] + ?.sources, ).toEqual(['/root-a/src/main/java']) expect( - resolved['/root-b/.socket.facts.json']!.projects[0]!.sources, + resolved['/root-b/.socket.facts.json']!.projects[0]!.outputs?.['main'] + ?.sources, ).toEqual(['/root-b/src/main/java']) }) diff --git a/src/commands/manifest/scripts/socket-facts.init.gradle b/src/commands/manifest/scripts/socket-facts.init.gradle index 11d0b7ee79..8d131a9303 100644 --- a/src/commands/manifest/scripts/socket-facts.init.gradle +++ b/src/commands/manifest/scripts/socket-facts.init.gradle @@ -10,6 +10,12 @@ import java.util.Collections // `Project.findProperty` only exists since Gradle 2.13; fall back to hasProperty/property for older Gradle. gradle.ext.socketProp = { proj, name -> proj.hasProperty(name) ? proj.property(name) : null } +// String.uncapitalize() needs Groovy 2.4.8; Gradle 1.12 bundles Groovy 1.8. +gradle.ext.socketDecap = { String n -> n ? n[0].toLowerCase() + n.substring(1) : n } +// The first of the named properties `obj` has, or null. +gradle.ext.socketFirstProp = { obj, List names -> + names.findResult { n -> try { obj."$n" } catch (Exception ignore) { null } } +} // `-Psocket.excludePaths` → PRE-COMPILED, comma-joined, anchored regex pattern sources, used only // to skip whole excluded subprojects (source-file-level exclusion is left to the reachability @@ -79,6 +85,11 @@ gradle.ext.socketFactsState = [ projectGaByPath : Collections.synchronizedMap([:]), // Project path -> "group:name:version", the coordinate a failed project dependency is reported by. projectGavByPath : Collections.synchronizedMap([:]), + // Project path -> output set -> the project's other output sets its classpath holds. + projectOutputSets : Collections.synchronizedMap([:]), + // Project path -> configuration name -> [set:, role: runtime|compile]: the output set whose classpath + // the configuration resolves. + classpathRoles : Collections.synchronizedMap([:]), ] // Capture every project's identity before collectors run so they can recognize intra-project deps @@ -115,75 +126,166 @@ gradle.projectsEvaluated { g -> // ALL source sets are emitted (not just main/test) so custom ones (integrationTest, // functionalTest, ...) reach the analysis; source-file-level exclusion is coana's job. // `classesDirs` (Gradle 4+) falls back to the legacy single `classesDir` on older Gradle. - def sources = [] as Set - def targets = [] as Set + // Output set -> its source and class dirs (--with-files only) and the project's other output sets its + // classpath holds. + def outputs = [:] + def output = { String set -> outputs.get(set, [sources: [] as Set, targets: [] as Set, includes: []]) } + def withFiles = g.ext.socketProp.call(p, 'socket.withFiles')?.toString()?.toLowerCase() == 'true' + def addDirs = { String set, String kind, dirs -> if (withFiles) output(set)[kind].addAll(dirs*.absolutePath) } + def roles = [:] + def classpathOf = { String set, String runtimeName, String compileName -> + output(set) + if (runtimeName) roles[runtimeName] = [set: set, role: 'runtime'] + if (compileName) roles[compileName] = [set: set, role: 'compile'] + } + def holds = { String set, List others -> + def o = output(set) + o.includes = (o.includes + others.findAll { it != null && it != set }).unique() + } + // The output sets wired into a classpath file collection directly (test's holds main's output). + def wiredOutputs = { fc -> + def found = [] + def seen = [] as Set + def walk + walk = { x -> + if (x == null || !seen.add(System.identityHashCode(x))) return + if (x instanceof org.gradle.api.tasks.SourceSetOutput) { found << x; return } + if (x instanceof Collection || x instanceof Object[]) { x.each { walk(it) }; return } + try { if (x.metaClass.respondsTo(x, 'getFrom')) x.from.each { walk(it) } } catch (Exception ignore) {} + } + walk(fc) + found + } try { if (p.hasProperty('sourceSets')) { p.sourceSets.each { ss -> if (ss != null) { - ss.allSource.srcDirs.each { d -> sources << d.absolutePath } + classpathOf(ss.name, + g.ext.socketFirstProp.call(ss, ['runtimeClasspathConfigurationName', 'runtimeConfigurationName']), + g.ext.socketFirstProp.call(ss, ['compileClasspathConfigurationName', 'compileConfigurationName'])) + addDirs(ss.name, 'sources', ss.allSource.srcDirs) try { - ss.output.classesDirs.each { d -> targets << d.absolutePath } + addDirs(ss.name, 'targets', ss.output.classesDirs) } catch (Exception e1) { - try { targets << ss.output.classesDir.absolutePath } catch (Exception ignore) {} + try { addDirs(ss.name, 'targets', [ss.output.classesDir]) } catch (Exception ignore) {} } } } + p.sourceSets.each { ss -> + try { + holds(ss.name, wiredOutputs(ss.runtimeClasspath).collect { out -> p.sourceSets.find { it.output.is(out) }?.name }) + } catch (Exception ignore) {} + } } } catch (Exception ignore) {} // Kotlin compilations for the JVM (kotlin-jvm, KMP `jvm()`, Android): Kotlin source sets, shared // ones like commonMain included, are not java source sets. Untyped: KGP is not on this classpath. + // A compilation takes its java source set's output set (kotlin-jvm), its variant's (Android) or its + // target and name (KMP: jvm, jvmTest), and holds its associated compilations (jvmTest: jvm). def kotlin = p.extensions.findByName('kotlin') if (kotlin != null) { try { + def setOf = { t, c, String runtimeName -> + roles[runtimeName]?.set ?: (t.platformType.name() == 'androidJvm' || !t.name ? c.name : + (c.name == 'main' ? t.name : t.name + c.name.capitalize())) + } (kotlin.hasProperty('targets') ? kotlin.targets : [kotlin.target]).each { t -> try { if (t.platformType.name() in ['jvm', 'androidJvm']) { t.compilations.each { c -> + def runtimeName = g.ext.socketFirstProp.call(c, ['runtimeDependencyConfigurationName']) + def set = setOf(t, c, runtimeName) + classpathOf(set, runtimeName, g.ext.socketFirstProp.call(c, ['compileDependencyConfigurationName'])) c.allKotlinSourceSets.each { ks -> - sources.addAll(ks.kotlin.srcDirs*.absolutePath) - sources.addAll(ks.resources.srcDirs*.absolutePath) + addDirs(set, 'sources', ks.kotlin.srcDirs) + addDirs(set, 'sources', ks.resources.srcDirs) } - targets.addAll(c.output.classesDirs*.absolutePath) + addDirs(set, 'targets', c.output.classesDirs) + def associated = g.ext.socketFirstProp.call(c, ['allAssociatedCompilations', 'associatedCompilations', 'associateWith']) + holds(set, ((associated ?: []) as List).collect { a -> setOf(t, a, null) }) } } } catch (Exception ignore) {} } } catch (Exception ignore) {} } - // AGP keeps its source sets in the `android` extension and leaves the java ones empty. + // AGP keeps its source sets in the `android` extension and leaves the java ones empty. Output sets + // are variants (debug, debugUnitTest, debugAndroidTest); a test variant holds the variant it tests. def android = p.extensions.findByName('android') if (android != null) { - try { - android.sourceSets.each { ss -> + def destinationOf = { task -> + g.ext.socketFirstProp.call(task, ['destinationDirectory'])?.get()?.asFile ?: g.ext.socketFirstProp.call(task, ['destinationDir']) + } + def addSources = { String set, provider -> + ['javaDirectories', 'resourcesDirectories', 'kotlinDirectories'].each { prop -> + try { addDirs(set, 'sources', provider."$prop") } catch (Exception ignore) {} + } + } + // The variant API (AGP before 9) names each variant's source sets, classpaths, compile task and + // tested variant. Kotlin class dirs come from the compilations above. + def variants = [] + ['applicationVariants', 'libraryVariants', 'testVariants', 'unitTestVariants'].each { prop -> + try { if (android.hasProperty(prop)) variants.addAll(android."$prop") } catch (Exception ignore) {} + } + if (variants) { + variants.each { v -> try { - sources.addAll(ss.java.srcDirs*.absolutePath) - sources.addAll(ss.resources.srcDirs*.absolutePath) - // Last: older AGP has no `kotlin`. - sources.addAll(ss.kotlin.srcDirs*.absolutePath) + classpathOf(v.name, v.runtimeConfiguration?.name, v.compileConfiguration?.name) + v.sourceSets.each { sp -> addSources(v.name, sp) } + def compile = g.ext.socketFirstProp.call(v, ['javaCompileProvider'])?.get() ?: g.ext.socketFirstProp.call(v, ['javaCompile']) + if (compile != null) addDirs(v.name, 'targets', [destinationOf(compile)].findAll()) + holds(v.name, [g.ext.socketFirstProp.call(v, ['testedVariant'])?.name]) } catch (Exception ignore) {} } - } catch (Exception ignore) {} - // Class dirs come from the compile tasks: AGP's built-in Kotlin (AGP 9) leaves the Kotlin - // compilations' classesDirs empty. - def compileTypes = [JavaCompile] - if (kotlin != null) { - try { - compileTypes << Class.forName('org.jetbrains.kotlin.gradle.tasks.KotlinCompileTool', false, kotlin.getClass().classLoader) - } catch (Throwable ignore) {} - } - compileTypes.each { type -> - try { - p.tasks.withType(type).each { t -> - try { - targets << t.destinationDirectory.get().asFile.absolutePath - } catch (Exception e1) { - try { targets << t.destinationDir.absolutePath } catch (Exception ignore) {} + } else { + // AGP 9 has no variant API, so AGP's naming: class dirs from the compile tasks (its built-in Kotlin + // leaves classesDirs empty), source sets to the variants of their kind whose name has their words. + def compileTypes = [JavaCompile] + if (kotlin != null) { + try { + compileTypes << Class.forName('org.jetbrains.kotlin.gradle.tasks.KotlinCompileTool', false, kotlin.getClass().classLoader) + } catch (Throwable ignore) {} + } + def androidSets = [] as LinkedHashSet + compileTypes.each { type -> + try { + p.tasks.withType(type).each { t -> + def m = t.name =~ /^compile(.+?)(JavaWithJavac|Kotlin)$/ + if (!m.matches()) return + def set = g.ext.socketDecap.call(m.group(1)) + androidSets << set + classpathOf(set, set + 'RuntimeClasspath', set + 'CompileClasspath') + try { addDirs(set, 'targets', [destinationOf(t)].findAll()) } catch (Exception ignore) {} } - } - } catch (Exception ignore) {} + } catch (Exception ignore) {} + } + def kinds = [androidTest: 'AndroidTest', testFixtures: 'TestFixtures', test: 'UnitTest'] + androidSets.each { set -> + def kind = kinds.values().find { set.endsWith(it) && set.length() > it.length() } + def tested = kind ? set.substring(0, set.length() - kind.length()) : null + if (androidSets.contains(tested)) holds(set, [tested]) + } + if (withFiles) { + def words = { String n -> (n =~ /[A-Z]?[a-z0-9]+/).collect { it.toLowerCase() } } + def setWords = androidSets.collectEntries { [it, words(it)] } + try { + android.sourceSets.each { ss -> + def kind = kinds.find { k, v -> ss.name.startsWith(k) } + def scope = words(kind ? ss.name.substring(kind.key.length()) : (ss.name == 'main' ? '' : ss.name)) + androidSets.findAll { v -> + (kind ? v.endsWith(kind.value) : !kinds.values().any { v.endsWith(it) }) && setWords[v].containsAll(scope) + }.each { set -> + ['java', 'resources', 'kotlin'].each { prop -> + try { addDirs(set, 'sources', ss."$prop".srcDirs) } catch (Exception ignore) {} + } + } + } + } catch (Exception ignore) {} + } } } + g.socketFactsState.projectOutputSets[p.path] = outputs.collectEntries { set, o -> [set, o.includes] } + g.socketFactsState.classpathRoles[p.path] = roles // `buildFile` is the configured script, absent on disk for a project configured from the root or // a convention plugin. Still emitted: a direct dependency's mark must name its subproject. def buildFiles = [] @@ -197,8 +299,9 @@ gradle.projectsEvaluated { g -> name : p.name, version: (p.version ?: '').toString(), dir : rel(p.projectDir), - sources: (sources as List).sort(), - targets: (targets as List).sort(), + outputs: outputs.collectEntries { set, o -> + [set, [sources: (o.sources as List).sort(), targets: (o.targets as List).sort(), includes: o.includes]] + }, buildFiles: buildFiles, excluded: excluded, ]) @@ -283,14 +386,25 @@ allprojects { project -> } // A node is created once; its prod flag accumulates (OR) across the configs that reach it. - def upsertNode = { Map coord, boolean isProd, String projPath -> + // The output set a dependency on a build project brings in, by the variant it selected: + // `[]RuntimeElements`/`ApiElements`, or `default` and the like for the main one. + def outputSetOf = { String projPath, String targetConfig -> + def m = (targetConfig ?: '') =~ /^(.+?)(RuntimeElements|ApiElements)$/ + def set = m.matches() ? gradle.ext.socketDecap.call(m.group(1)) : 'main' + state.projectOutputSets[projPath]?.containsKey(set) ? set : 'main' + } + + def upsertNode = { Map coord, boolean isProd, String projPath, String outputSet = null -> def id = coordId(coord) synchronized (nodes) { def node = nodes[id] if (node == null) { - node = [coord: coord, children: [] as Set, prod: false, project: projPath] + node = [coord: coord, children: [] as Set, prod: false, project: projPath, outputSets: [] as LinkedHashSet] nodes[id] = node } + if (outputSet != null) { + node.outputSets << outputSet + } if (isProd) { node.prod = true } @@ -337,6 +451,7 @@ allprojects { project -> def gav = "${dep.moduleGroup ?: ''}:${dep.moduleName}:${dep.moduleVersion ?: ''}".toString() def shouldMaterialize = withFiles && (populateGavs == null || populateGavs.contains(gav)) def projPath = projectByGav[gav] + def outputSet = projPath == null ? null : outputSetOf(projPath, dep.configuration) def artifacts def artifactsError = null try { @@ -360,7 +475,7 @@ allprojects { project -> version : dep.moduleVersion ?: '', classifier: '', ext : ext, - ], isProd, projPath) + ], isProd, projPath, outputSet) } } else { artifacts.each { a -> @@ -375,7 +490,7 @@ allprojects { project -> version : dep.moduleVersion ?: '', classifier: a.classifier ?: '', ext : ext, - ], isProd, projPath) + ], isProd, projPath, outputSet) producedIds << aid // `a.file` downloads the artifact if not already cached, so scoping avoids fetching // artifacts the SBOM doesn't reference. Per-artifact try/catch: a single download can @@ -862,9 +977,14 @@ rootProject { rp -> rec(['project', pi.path, pi.group ?: '', pi.name, pi.version ?: '', pi.dir]) pi.buildFiles.each { f -> rec(['projectBuild', pi.path, f]) } } + if (!pi.excluded || withFilesProjects) { + pi.outputs.each { set, o -> rec(['outputSet', pi.path, set, o.includes.join(',')]) } + } if (withFilesProjects) { - pi.sources.each { s -> rec(['projectSrc', pi.path, s]) } - pi.targets.each { t -> rec(['projectTgt', pi.path, t]) } + pi.outputs.each { set, o -> + o.sources.each { s -> rec(['projectSrc', pi.path, s, set]) } + o.targets.each { t -> rec(['projectTgt', pi.path, t, set]) } + } } } @@ -875,11 +995,13 @@ rootProject { rp -> int rootIdx = 0 perSub.each { rootKey, tree -> def rootId = (rootIdx++).toString() - rec(['root', rootId, tree.projectPath, tree.config, tree.prod ? '1' : '0']) + def classpath = state.classpathRoles[tree.projectPath]?.get(tree.config) + rec(['root', rootId, tree.projectPath, tree.config, tree.prod ? '1' : '0', classpath?.set ?: '', classpath?.role ?: '']) tree.nodes.each { coordId, node -> def c = node.coord rec(['node', rootId, coordId, c.groupId ?: '', c.artifactId ?: '', c.version ?: '', c.ext ?: '', - c.classifier ?: '', tree.direct.contains(coordId) ? '1' : '0', node.project ?: '']) + c.classifier ?: '', tree.direct.contains(coordId) ? '1' : '0', node.project ?: '', + (node.outputSets ?: []).join(',')]) node.children.each { childId -> rec(['edge', rootId, coordId, childId]) } def fs = tree.paths[coordId] if (fs) { (fs as List).sort().each { p -> rec(['file', rootId, coordId, p]) } } diff --git a/src/commands/manifest/scripts/socket-facts.plugin.scala b/src/commands/manifest/scripts/socket-facts.plugin.scala index 6a736865d4..3d1eb03d87 100644 --- a/src/commands/manifest/scripts/socket-facts.plugin.scala +++ b/src/commands/manifest/scripts/socket-facts.plugin.scala @@ -73,6 +73,8 @@ object SocketFactsPlugin extends AutoPlugin { // Real artifact ext per build module, so an ext-less inter-project dep gets its true coordinate. val moduleExts = buildModuleExts(allRefs, extracted) + val outputSets: Map[ProjectRef, Seq[(String, Configuration)]] = + allRefs.map(r => r -> sourceConfigs(r, extracted)).toMap // Every project: `update` also runs each upstream project's update, which would fail strictly. lazy val lenientState = extracted.append(allRefs.map(r => updateConfiguration.in(r) ~= (c => withMissingOk(c))), st) @@ -93,15 +95,15 @@ object SocketFactsPlugin extends AutoPlugin { runUpdateResilient(updateTaskName, ref, extracted, st, lenientState, matcher, failures, unscannable, resolver) .foreach { case (report, unresolved, ivyUnfetched, updateFailed) => foldReport(report, unresolved, ref, extracted, matcher, scannedConfigs, withFiles, populateScope, moduleExts, - resolver, ivyUnfetched, updateFailed, failures).foreach { + outputSets, resolver, ivyUnfetched, updateFailed, failures).foreach { case (rootKey, tree) => perSub(rootKey) = tree } } } } - val moduleDirs: Map[String, (Seq[String], Seq[String])] = - if (withFiles) buildModuleDirs(allRefs, extracted, st) else Map.empty + val moduleDirs: Map[String, Seq[(String, Seq[String], Seq[String])]] = + if (withFiles) buildModuleDirs(allRefs, outputSets, extracted, st) else Map.empty // Where sbt recorded the project's own settings as defined (the root build.sbt for a subproject // defined there); positions outside the build are sbt defaults, plugins or our injected base. @@ -141,11 +143,19 @@ object SocketFactsPlugin extends AutoPlugin { (buildFilesByRef.getOrElse(ref, Nil) ++ baseBuildSbt(baseDir, rootCanonPath).map(relOf)).distinct.sorted .foreach(f => rec("projectBuild", ref.project, f)) } + // A configuration's classpath holds the code of the configurations it extends (Test: Compile). + if (!isExcludedRef(ref) || withFiles) { + val sets = outputSets(ref) + sets.foreach { case (set, c) => + val held = Classpaths.allConfigs(c).map(_.name).toSet + rec("outputSet", ref.project, set, sets.collect { case (o, oc) if oc.name != c.name && held(oc.name) => o }.mkString(",")) + } + } if (withFiles) { - moduleDirs.get(mid.organization + ":" + mid.name + ":" + ver).foreach { - case (sources, targets) => - sources.foreach(s => rec("projectSrc", ref.project, s)) - targets.foreach(t => rec("projectTgt", ref.project, t)) + moduleDirs.getOrElse(mid.organization + ":" + mid.name + ":" + ver, Nil).foreach { + case (set, sources, targets) => + sources.foreach(s => rec("projectSrc", ref.project, s, set)) + targets.foreach(t => rec("projectTgt", ref.project, t, set)) } } } @@ -157,12 +167,12 @@ object SocketFactsPlugin extends AutoPlugin { case (_, tree) => val rootId = rootIdx.toString rootIdx += 1 - rec("root", rootId, tree.projectKey, tree.config, if (tree.prod) "1" else "0") + rec("root", rootId, tree.projectKey, tree.config, if (tree.prod) "1" else "0", tree.outputSet, tree.role) tree.nodes.foreach { case (coordId, node) => val c = node.coord rec("node", rootId, coordId, c.org, c.name, c.version, c.ext, c.classifier, if (node.direct) "1" else "0", - node.project) + node.project, node.outputSets.mkString(",")) node.children.foreach(ch => rec("edge", rootId, coordId, ch)) node.targets.foreach(p => rec("file", rootId, coordId, p)) if (node.unfetched) rec("unfetched", rootId, coordId) @@ -202,27 +212,36 @@ object SocketFactsPlugin extends AutoPlugin { }.toMap } - // Absolute source roots + compiled-output dirs per build module, keyed by GAV. --with-files only; - // absolute because reachability locates an internal module's code on THIS machine (no registry jar). - // ALL of the project's configurations are scanned (not just Compile/Test) so custom ones - // (IntegrationTest, ...) reach the analysis; source-file-level exclusion is coana's job. + // A project's code by output set: Compile's as `main`, and each configuration with a class directory + // of its own (Test, IntegrationTest, ...) under its name. + private def sourceConfigs(ref: ProjectRef, extracted: Extracted): Seq[(String, Configuration)] = { + val configs = extracted.getOpt(thisProject.in(ref)).map(_.configurations).getOrElse(Seq(Compile, Test)) + val mainDir = extracted.getOpt(classDirectory.in(ref).in(Compile)) + ("main" -> Compile) +: configs.filter { c => + c.name != Compile.name && extracted.getOpt(classDirectory.in(ref).in(c)).exists(d => !mainDir.exists(_ == d)) + }.map(c => c.name -> c) + } + + // Absolute source roots + compiled-output dirs per build module and output set, keyed by GAV. + // --with-files only; absolute because reachability locates an internal module's code on THIS machine + // (no registry jar). Source-file-level exclusion is coana's job. private def buildModuleDirs( allRefs: Seq[ProjectRef], + outputSets: Map[ProjectRef, Seq[(String, Configuration)]], extracted: Extracted, state: State - ): Map[String, (Seq[String], Seq[String])] = { + ): Map[String, Seq[(String, Seq[String], Seq[String])]] = { allRefs.map { ref => val mid = rootIdOf(extracted, ref) val ver = if (mid.revision == null) "" else mid.revision - val configs: Seq[Configuration] = - extracted.getOpt(thisProject.in(ref)).map(_.configurations).getOrElse(Seq(Compile, Test)) - val sources = (configs.flatMap(c => extracted.getOpt(sourceDirectories.in(ref).in(c)).getOrElse(Nil)) ++ - baseDirSources(ref, configs, extracted, state)) - .map(_.getAbsolutePath).distinct.sorted - val targets = configs - .flatMap(c => extracted.getOpt(classDirectory.in(ref).in(c))) - .map(_.getAbsolutePath).distinct.sorted - (mid.organization + ":" + mid.name + ":" + ver) -> ((sources, targets)) + val sets = outputSets(ref).map { case (set, c) => + val sources = (extracted.getOpt(sourceDirectories.in(ref).in(c)).getOrElse(Nil) ++ + baseDirSources(ref, Seq(c), extracted, state)) + .map(_.getAbsolutePath).distinct.sorted + val targets = extracted.getOpt(classDirectory.in(ref).in(c)).toSeq.map(_.getAbsolutePath) + (set, sources, targets) + } + (mid.organization + ":" + mid.name + ":" + ver) -> sets }.toMap } @@ -489,6 +508,7 @@ object SocketFactsPlugin extends AutoPlugin { withFiles: Boolean, populateScope: Option[Set[String]], moduleExts: Map[String, String], + outputSets: Map[ProjectRef, Seq[(String, Configuration)]], resolver: Resolver, ivyUnfetched: IvyUnfetched, // sbt's own update failed on artifacts alone, so one it could not fetch is a failure even @@ -554,6 +574,19 @@ object SocketFactsPlugin extends AutoPlugin { case Some(gavs) => gavs.contains(gavKey(m)) } + // The configuration sbt resolves a configuration's classpath for: the first of its internal one + // (compile-internal, ...), Compile and Default the report has (Defaults.findClasspathConfig). + val reported = report.configurations.map(confName).toSet + def classpathConf(c: Configuration): Option[String] = { + val internal = extracted.getOpt(internalConfigurationMap.in(ref)).map(_(c).name).getOrElse(c.name) + Seq(internal, Compile.name, "default").find(reported) + } + // Each output set's runtime classpath, and Compile's, which adds the compile-only (Provided) entries. + val classpathRoles: Map[String, (String, String)] = + outputSets(ref).flatMap { case (set, c) => + classpathConf(if (set == "main") Runtime else c).map(_ -> ((set, "runtime"))) + }.toMap ++ classpathConf(Compile).map(_ -> (("main", "compile"))) + report.configurations.foreach { cr => val cfg = confName(cr) if (matcher(cfg)) { @@ -566,13 +599,17 @@ object SocketFactsPlugin extends AutoPlugin { confsOf(ref).find(_.name == cfg).foreach { conf => // Keyed by project id, as projects need not have distinct coordinates. + // A dependency on a build project brings in the output sets of the configurations it reaches. def buildModuleNode(r: ProjectRef): String = { midToIds.getOrElseUpdate(moduleKey(rootIdOf(extracted, r)), mutable.LinkedHashSet.empty[String]) += r.project - nodes.getOrElseUpdate(r.project, { + val node = nodes.getOrElseUpdate(r.project, { val node = new Node(buildModuleCoord(r)) node.project = r.project node }) + val confs = reachedConfs.getOrElse(r.project, Set.empty[String]) + node.outputSets ++= outputSets.getOrElse(r, Nil) + .collect { case (set, c) if confs(c.name) => set } r.project } val (edges, reached) = interProjectEdges(conf) @@ -632,7 +669,10 @@ object SocketFactsPlugin extends AutoPlugin { } } - if (nodes.nonEmpty) perRoot(ref.project + "::" + cfg) = new RootTree(ref.project, cfg, prod, nodes) + if (nodes.nonEmpty) { + val (set, role) = classpathRoles.getOrElse(cfg, ("", "")) + perRoot(ref.project + "::" + cfg) = new RootTree(ref.project, cfg, prod, set, role, nodes) + } } } perRoot @@ -1000,6 +1040,8 @@ object SocketFactsPlugin extends AutoPlugin { var direct = false // Id of the build project this node is, when it is one. var project = "" + // That project's output sets this dependency brings in. + val outputSets = mutable.LinkedHashSet.empty[String] // External artifact's resolved jar(s); --with-files only. val targets = mutable.TreeSet.empty[String] // An expected artifact was not fetched; --with-files only. @@ -1013,6 +1055,9 @@ object SocketFactsPlugin extends AutoPlugin { val projectKey: String, val config: String, val prod: Boolean, + // The output set whose runtime or compile classpath this root is, or "". + val outputSet: String, + val role: String, val nodes: mutable.LinkedHashMap[String, Node] ) } diff --git a/src/commands/manifest/scripts/test/maven-compat/assert-classpath.py b/src/commands/manifest/scripts/test/maven-compat/assert-classpath.py index a6062dff0f..15d0a60fa9 100644 --- a/src/commands/manifest/scripts/test/maven-compat/assert-classpath.py +++ b/src/commands/manifest/scripts/test/maven-compat/assert-classpath.py @@ -1,35 +1,41 @@ #!/usr/bin/env python3 # Assert the `classpath` records the Coana Maven core extension emitted for the smoke project's app -# modules: the order Maven builds their classpaths in (a pre-order walk of the resolved graph, filtered -# by scope, the test scope moved last). app keeps demo.ext:libdep, which Maven leaves only under lib's -# test-jar node while the walk reaches lib's jar node first; app2 reaches lib's node through the -# test-jar first and still puts lib on the main classpath. Matches on groupId:artifactId, so versions -# can change freely. +# modules: each classpath in the order Maven builds it (a pre-order walk of the resolved graph, filtered +# by scope), with the output sets each reactor module's artifacts bring in. app keeps demo.ext:libdep, +# which Maven leaves only under lib's test-jar node while the walk reaches lib's jar node first. +# Matches on groupId:artifactId, so versions can change freely. import sys rows = [l.rstrip('\n').split('\t') for l in open(sys.argv[1]) if l.strip()] -root_project = {r[1]: r[2] for r in rows if r[0] == 'root'} EXPECTED = { - 'demo:app:1.0': [ - 'demo:mid', 'demo:lib', 'demo.ext:tool', 'demo.ext:shared', 'demo.ext:conflict-prod', - 'demo.ext:libdep', 'demo.ext:harness', 'demo.ext:harness-core', 'demo.ext:conflict-test', - ], - 'demo:app2:1.0': ['demo.ext:libdep', 'demo:mid', 'demo:lib'], + 'demo:app:1.0': { + 'main': [ + ('demo:mid', 'main'), ('demo:lib', 'main'), ('demo.ext:tool', ''), ('demo.ext:shared', ''), + ('demo.ext:conflict-prod', ''), ('demo.ext:libdep', ''), + ], + 'test': [ + ('demo:mid', 'main'), ('demo:lib', 'main,test'), ('demo.ext:tool', ''), ('demo.ext:harness', ''), + ('demo.ext:harness-core', ''), ('demo.ext:conflict-test', ''), ('demo.ext:shared', ''), + ('demo.ext:conflict-prod', ''), ('demo.ext:libdep', ''), + ], + }, + 'demo:app2:1.0': { + 'main': [('demo.ext:libdep', ''), ('demo:mid', 'main'), ('demo:lib', 'main')], + 'test': [('demo:lib', 'test,main'), ('demo.ext:libdep', ''), ('demo:mid', 'main')], + }, } errors = [] -for project, expected in EXPECTED.items(): - nodes = {r[2] for r in rows if r[0] == 'node' and root_project.get(r[1]) == project} - classpath = [r[2] for r in rows if r[0] == 'classpath' and r[1] == project] - if sorted(classpath) != sorted(nodes): - errors.append(f"{project}: classpath {sorted(classpath)} != its nodes {sorted(nodes)}") - got = [':'.join(c.split(':')[:2]) for c in classpath] - if got != expected: - errors.append(f"{project}: classpath order {got} != {expected}") +for project, classpaths in EXPECTED.items(): + for name, expected in classpaths.items(): + got = [(':'.join(r[3].split(':')[:2]), r[4] if len(r) > 4 else '') + for r in rows if r[0] == 'classpath' and r[1] == project and r[2] == name] + if got != expected: + errors.append(f"{project} {name}: {got} != {expected}") if errors: print("FAIL:") for e in errors: print(" -", e) sys.exit(1) -print(f"PASS (classpath): {', '.join(EXPECTED)} in Maven's order, test scope last") +print(f"PASS (classpath): {', '.join(EXPECTED)} main and test classpaths in Maven's order") diff --git a/src/commands/manifest/scripts/test/maven-compat/smoke-test.sh b/src/commands/manifest/scripts/test/maven-compat/smoke-test.sh index a17699562b..91f5ad01a7 100755 --- a/src/commands/manifest/scripts/test/maven-compat/smoke-test.sh +++ b/src/commands/manifest/scripts/test/maven-compat/smoke-test.sh @@ -8,7 +8,7 @@ # inter-module edge lines up with its `project` record); # - resolves a scope conflict so the retained node stays reachable from its parent root; # - every component is reachable from a direct dependency of its own root (no orphaned components); -# - states each module's classpath in Maven's order, test scope last, keeping the dependencies Maven +# - states each module's main and test classpaths in Maven's order, keeping the dependencies Maven # leaves only under a sibling module's test-jar (assert-classpath.py); # - materializes resolved external jars under -Dsocket.withFiles; # - scopes that materialization to -Dsocket.populateFilesFor (a newline-delimited GAV file).