diff --git a/.gitignore b/.gitignore index 3dd4c3ebf..3c875633e 100644 --- a/.gitignore +++ b/.gitignore @@ -40,6 +40,11 @@ test/fixtures/commands/fix/e2e-test-py-temp-* /src/commands/manifest/scripts/test/gradle-compat/project/localrepo/ /src/commands/manifest/scripts/test/gradle-compat/project/records.tsv /src/commands/manifest/scripts/test/gradle-compat/project/workspaces-records.tsv +/src/commands/manifest/scripts/test/gradle-compat/order/.gradle/ +/src/commands/manifest/scripts/test/gradle-compat/order/build/ +/src/commands/manifest/scripts/test/gradle-compat/order/classpath.txt +/src/commands/manifest/scripts/test/gradle-compat/order/localrepo/ +/src/commands/manifest/scripts/test/gradle-compat/order/records.tsv /src/commands/manifest/scripts/test/maven-compat/project/localrepo/ /src/commands/manifest/scripts/test/maven-compat/project/records.tsv /src/commands/manifest/scripts/test/maven-compat/project/target/ diff --git a/src/commands/manifest/run-manifest-facts.test.mts b/src/commands/manifest/run-manifest-facts.test.mts index 45b45cf77..543bb2542 100644 --- a/src/commands/manifest/run-manifest-facts.test.mts +++ b/src/commands/manifest/run-manifest-facts.test.mts @@ -34,7 +34,8 @@ function okResult(buildRoot: string): ManifestRunResult { }, artifactPaths: { pathsById: new Map(), - classpathByProject: new Map(), + outputsByProject: new Map(), + classpathsByProject: new Map(), directDependenciesByProject: new Map(), }, stderr: '', diff --git a/src/commands/manifest/scripts/assemble.mts b/src/commands/manifest/scripts/assemble.mts index 81a7d1a19..0a5861727 100644 --- a/src/commands/manifest/scripts/assemble.mts +++ b/src/commands/manifest/scripts/assemble.mts @@ -1,16 +1,25 @@ import { existsSync } from 'node:fs' -import { - type ResolvedArtifactPaths, - type SocketFactsSbom, - type SocketFactsSbomComponent, - type SocketFactsSbomDependency, - type SocketFactsSbomMetadata, - type SocketFactsSbomProject, -} from './facts.mts' import { moduleKey } from './resolution-report.mts' -import type { ParsedRecords, RawCoord, RawProject } from './records.mts' +import type { + ClasspathEntry, + OutputPaths, + ResolvedArtifactPaths, + SocketFactsSbom, + SocketFactsSbomComponent, + SocketFactsSbomDependency, + SocketFactsSbomMetadata, + SocketFactsSbomProject, +} from './facts.mts' + +import type { + ParsedRecords, + RawCoord, + RawProject, + RawRoot, + StatedClasspathEntry, +} from './records.mts' import type { ResolutionReport } from './resolution-report.mts' const PURL_TYPE_MAVEN = 'maven' @@ -45,6 +54,9 @@ type SubprojectGraph = { type PerRoot = { projectKey: string prod: boolean + outputSet: string + role: RawRoot['role'] + // In the root's classpath order. nodes: Map } @@ -53,6 +65,7 @@ type RootNode = { children: string[] direct: boolean project: string + outputSets: string[] targets: string[] unfetched: boolean } @@ -93,6 +106,7 @@ export function assembleFacts( [...parsed.projects.values()], [...parsed.excludedProjects.values()], graphs, + buildClasspaths(parsed, perRoot), fileExists, ), } @@ -119,11 +133,17 @@ function buildPerRoot(parsed: ParsedRecords): Map { children: [], direct: false, project: n.project, + outputSets: [], targets: [], unfetched: false, } nodes.set(id, node) } + for (const set of n.outputSets) { + if (!node.outputSets.includes(set)) { + node.outputSets.push(set) + } + } node.direct ||= n.direct node.unfetched ||= n.unfetched node.targets.push(...n.targets) @@ -139,14 +159,20 @@ function buildPerRoot(parsed: ParsedRecords): Map { parent.children.push(childId) } } - out.set(rootId, { projectKey: r.projectKey, prod: r.prod, nodes }) + out.set(rootId, { + projectKey: r.projectKey, + prod: r.prod, + outputSet: r.outputSet, + role: r.role, + nodes, + }) } return out } // Components are merged by id across every resolution root; which ids belong -// to which subproject is kept separately (classpathByProject) for -// reachability, which needs each subproject's exact classpath. +// to which subproject is kept separately (classpathsByProject) for +// reachability. function mergeById(perRoot: Map): Map { const finalNodes = new Map() for (const { nodes, prod } of perRoot.values()) { @@ -395,24 +421,151 @@ function treeRoots( return roots } -function byProject( - projects: RawProject[], - graphs: Map, - ids: (graph: SubprojectGraph) => Iterable, -): Map { - return new Map( - projects.map(p => { - const g = graphs.get(p.projectKey) - return [p.projectKey, g ? [...ids(g)].sort() : []] - }), +// A dependency on a build project names the output sets it brings in; without +// one it is a plain dependency, which brings the project's main code. +function entriesOf( + id: string, + project: string, + outputSets: string[], +): ClasspathEntry[] { + if (!project) { + return [{ id }] + } + return (outputSets.length ? outputSets : ['main']).map(outputs => ({ + project: id, + outputs, + })) +} + +// The project's own code goes first, the way the build tools put it. +function ownEntries(project: RawProject, outputSet: string): ClasspathEntry[] { + return [outputSet, ...(project.outputSets.get(outputSet) ?? [])].map( + outputs => ({ project: project.projectKey, outputs }), ) } +function uniqueEntries(entries: ClasspathEntry[]): ClasspathEntry[] { + const seen = new Set() + return entries.filter(e => { + const key = 'id' in e ? e.id : `${e.project}\u0000${e.outputs}` + if (seen.has(key)) { + return false + } + seen.add(key) + return true + }) +} + +// The runtime classpath, then the entries only the compile classpath has +// (provided, compileOnly), in the version the runtime one resolved, if any. +function derivedClasspath(roots: PerRoot[]): ClasspathEntry[] { + const ordered = [ + ...roots.filter(r => r.role === 'runtime'), + ...roots.filter(r => r.role === 'compile'), + ] + const versionOf = new Map() + const out: ClasspathEntry[] = [] + for (const { nodes } of ordered) { + for (const { 0: id, 1: node } of nodes) { + const module = node.project + ? `project ${node.project}` + : `${node.coord.group}:${node.coord.name}` + let version = versionOf.get(module) + if (version === undefined) { + version = node.coord.version + versionOf.set(module, version) + } + if (version === node.coord.version) { + out.push(...entriesOf(id, node.project, node.outputSets)) + } + } + } + return out +} + +function statedClasspath( + projectKey: string, + stated: StatedClasspathEntry[], + projectOf: Map, +): ClasspathEntry[] { + return stated.flatMap(({ coordId, outputSets }) => { + const project = projectOf.get(coordId) + if (project === undefined) { + throw new Error( + `The classpath of ${projectKey} names ${coordId}, which none of its resolution roots resolved`, + ) + } + return entriesOf(componentId(coordId, project), project, outputSets) + }) +} + +function buildClasspaths( + parsed: ParsedRecords, + perRoot: Map, +): Map> { + const rootIdsByProject = new Map() + for (const { 0: rootId, 1: r } of parsed.roots) { + let list = rootIdsByProject.get(r.projectKey) + if (!list) { + list = [] + rootIdsByProject.set(r.projectKey, list) + } + list.push(rootId) + } + const out = new Map>() + for (const { 0: projectKey, 1: project } of parsed.projects) { + const rootIds = rootIdsByProject.get(projectKey) ?? [] + // Every output set has a classpath, if only its own code. + const classpaths = new Map( + [...project.outputSets.keys()].map(name => [name, []]), + ) + const stated = parsed.classpaths.get(projectKey) + if (stated) { + const projectOf = new Map() + for (const id of rootIds) { + for (const { 0: coordId, 1: n } of parsed.roots.get(id)?.nodes ?? []) { + projectOf.set(coordId, n.project) + } + } + for (const { 0: name, 1: entries } of stated) { + classpaths.set(name, statedClasspath(projectKey, entries, projectOf)) + } + } else { + const bySet = new Map() + for (const id of rootIds) { + const r = perRoot.get(id) + if (r?.outputSet && r.role) { + let roots = bySet.get(r.outputSet) + if (!roots) { + roots = [] + bySet.set(r.outputSet, roots) + } + roots.push(r) + } + } + for (const { 0: name, 1: roots } of bySet) { + if (roots.some(r => r.role === 'runtime')) { + classpaths.set(name, derivedClasspath(roots)) + } + } + } + for (const { 0: name, 1: entries } of classpaths) { + classpaths.set( + name, + uniqueEntries([...ownEntries(project, name), ...entries]), + ) + } + out.set(projectKey, classpaths) + } + return out +} + function buildArtifactPaths( finalNodes: Map, projects: RawProject[], excludedProjects: RawProject[], graphs: Map, + classpaths: Map>, fileExists: (path: string) => boolean, ): ResolvedArtifactPaths { const pathsById: ResolvedArtifactPaths['pathsById'] = new Map() @@ -436,26 +589,34 @@ function buildArtifactPaths( pathsById.set( id, fn.resolutions.length && !complete.length - ? { sources: [] } - : { - sources: [], - targets: [...new Set(complete.flatMap(r => r.targets))].sort(), - }, + ? {} + : { targets: [...new Set(complete.flatMap(r => r.targets))].sort() }, ) } } // A project's own component shares its id, so this also covers dependency // edges onto a sibling project. + const outputsByProject = new Map>() for (const p of [...projects, ...excludedProjects]) { - pathsById.set(p.projectKey, { - sources: [...new Set(p.sources)].filter(fileExists).sort(), - targets: [...new Set(p.targets)].filter(fileExists).sort(), - }) + const sets = new Map() + for (const { 0: name, 1: o } of p.outputs) { + sets.set(name, { + sources: [...new Set(o.sources)].filter(fileExists).sort(), + targets: [...new Set(o.targets)].filter(fileExists).sort(), + }) + } + outputsByProject.set(p.projectKey, sets) } return { pathsById, - classpathByProject: byProject(projects, graphs, g => g.children.keys()), - directDependenciesByProject: byProject(projects, graphs, g => g.direct), + outputsByProject, + classpathsByProject: classpaths, + directDependenciesByProject: new Map( + projects.map(p => { + const g = graphs.get(p.projectKey) + return [p.projectKey, g ? [...g.direct].sort() : []] + }), + ), } } diff --git a/src/commands/manifest/scripts/assemble.test.mts b/src/commands/manifest/scripts/assemble.test.mts index b78766fc5..3be4eef28 100644 --- a/src/commands/manifest/scripts/assemble.test.mts +++ b/src/commands/manifest/scripts/assemble.test.mts @@ -4,7 +4,7 @@ import { assembleFacts } from './assemble.mts' import { parseRecords } from './records.mts' import { accumulateSidecar, serializeSidecar } from './sidecar.mts' -import type { SocketFactsSbom } from './facts.mts' +import type { ClasspathEntry, SocketFactsSbom } from './facts.mts' import type { SidecarAccumulator } from './sidecar.mts' // Each project's tree, written out with component ids; a repeat on the current @@ -32,9 +32,9 @@ function trees(facts: SocketFactsSbom): Record { const RECORDS = [ 'meta\tgradle\t8.0\t17', 'project\t:app\tcom.example\tapp\t1.0\t/abs/app', - 'projectSrc\t:app\t/abs/app/src/main/java', - 'projectTgt\t:app\t/abs/app/build/classes', - 'root\tr1\t:app\truntimeClasspath\t1', + 'projectSrc\t:app\t/abs/app/src/main/java\tmain', + 'projectTgt\t:app\t/abs/app/build/classes\tmain', + 'root\tr1\t:app\truntimeClasspath\t1\tmain\truntime', 'node\tr1\tcom.example:lib:jar:1.0\tcom.example\tlib\t1.0\tjar\t\t1', 'node\tr1\tcom.example:bom:2.0\tcom.example\tbom\t2.0\t\t\t1', 'file\tr1\tcom.example:lib:jar:1.0\t/abs/lib.jar', @@ -98,9 +98,19 @@ describe('records → assemble → sidecar', () => { version: '1.0', subprojectDir: '/abs/app', dependencies: ['com.example:bom:2.0', 'com.example:lib:jar:1.0'], - targets: ['/abs/app/build/classes'], - sources: ['/abs/app/src/main/java'], - classpath: ['com.example:bom:2.0', 'com.example:lib:jar:1.0'], + outputs: { + main: { + sources: ['/abs/app/src/main/java'], + targets: ['/abs/app/build/classes'], + }, + }, + classpaths: { + main: [ + { project: ':app', outputs: 'main' }, + { id: 'com.example:lib:jar:1.0' }, + { id: 'com.example:bom:2.0' }, + ], + }, }, ]) @@ -119,12 +129,12 @@ describe('records → assemble → sidecar', () => { const records = [ 'meta\tgradle\t8.10\t17', 'project\t:app\tg\tapp\t1\tapp', - 'projectSrc\t:app\t/abs/app/src', - 'projectTgt\t:app\t/abs/app/classes', + 'projectSrc\t:app\t/abs/app/src\tmain', + 'projectTgt\t:app\t/abs/app/classes\tmain', 'excludedProject\t:lib', - 'projectSrc\t:lib\t/abs/lib/src', - 'projectTgt\t:lib\t/abs/lib/classes', - 'root\t0\t:app\tcompileClasspath\t1', + 'projectSrc\t:lib\t/abs/lib/src\tmain', + 'projectTgt\t:lib\t/abs/lib/classes\tmain', + 'root\t0\t:app\truntimeClasspath\t1\tmain\truntime', 'node\t0\t:lib\tg\tlib\t1\tjar\t\t1\t:lib', ].join('\n') const { artifactPaths, facts } = assembleFacts(parseRecords(records), { @@ -132,11 +142,15 @@ describe('records → assemble → sidecar', () => { }) expect(facts.projects?.map(p => p.id)).toEqual([':app']) - expect(artifactPaths.pathsById.get(':lib')).toEqual({ + expect(artifactPaths.outputsByProject.get(':lib')?.get('main')).toEqual({ sources: ['/abs/lib/src'], targets: ['/abs/lib/classes'], }) - expect([...artifactPaths.classpathByProject.keys()]).toEqual([':app']) + expect([...artifactPaths.classpathsByProject.keys()]).toEqual([':app']) + expect(artifactPaths.classpathsByProject.get(':app')?.get('main')).toEqual([ + { project: ':app', outputs: 'main' }, + { project: ':lib', outputs: 'main' }, + ]) }) it('states no targets for an artifact that failed to resolve or is gone from disk', () => { const records = [ @@ -220,15 +234,16 @@ describe('records → assemble → sidecar', () => { 'meta\tgradle\t8.0\t17', 'project\t:a\tcom.example\ta\t1.0\ta', 'project\t:b\tcom.example\tb\t1.0\tb', - 'root\tr1\t:a\truntimeClasspath\t1', + 'root\tr1\t:a\truntimeClasspath\t1\tmain\truntime', 'node\tr1\tg:lib:jar:1\tg\tlib\t1\tjar\t\t1', 'node\tr1\tg:dep:jar:1\tg\tdep\t1\tjar\t\t0', 'edge\tr1\tg:lib:jar:1\tg:dep:jar:1', - 'root\tr2\t:b\truntimeClasspath\t1', + 'root\tr2\t:b\truntimeClasspath\t1\tmain\truntime', 'node\tr2\tg:lib:jar:1\tg\tlib\t1\tjar\t\t1', 'node\tr2\tg:dep:jar:2\tg\tdep\t2\tjar\t\t0', 'edge\tr2\tg:lib:jar:1\tg:dep:jar:2', - 'root\tr3\t:b\ttestRuntimeClasspath\t0', + 'outputSet\t:b\ttest\tmain', + 'root\tr3\t:b\ttestRuntimeClasspath\t0\ttest\truntime', 'node\tr3\tg:junit:jar:4\tg\tjunit\t4\tjar\t\t1', ].join('\n') const { artifactPaths, facts } = assembleFacts(parseRecords(records), { @@ -262,12 +277,25 @@ describe('records → assemble → sidecar', () => { p, ]), ) - expect(byName.get('a')?.classpath).toEqual(['g:dep:jar:1', 'g:lib:jar:1']) - expect(byName.get('b')?.classpath).toEqual([ - 'g:dep:jar:2', - 'g:junit:jar:4', - 'g:lib:jar:1', - ]) + expect(byName.get('a')?.classpaths).toEqual({ + main: [ + { project: ':a', outputs: 'main' }, + { id: 'g:lib:jar:1' }, + { id: 'g:dep:jar:1' }, + ], + }) + expect(byName.get('b')?.classpaths).toEqual({ + main: [ + { project: ':b', outputs: 'main' }, + { id: 'g:lib:jar:1' }, + { id: 'g:dep:jar:2' }, + ], + test: [ + { project: ':b', outputs: 'test' }, + { project: ':b', outputs: 'main' }, + { id: 'g:junit:jar:4' }, + ], + }) expect(byName.get('b')?.dependencies).toEqual([ 'g:junit:jar:4', 'g:lib:jar:1', @@ -417,17 +445,18 @@ describe('records → assemble → sidecar', () => { const records = [ 'meta\tgradle\t8.0\t17', 'project\t:a:util\tex\tutil\t1\ta/util', - 'projectSrc\t:a:util\t/abs/a/util/src', + 'projectSrc\t:a:util\t/abs/a/util/src\tmain', 'project\t:b:util\tex\tutil\t1\tb/util', - 'projectSrc\t:b:util\t/abs/b/util/src', + 'projectSrc\t:b:util\t/abs/b/util/src\tmain', + 'projectSrc\t:b:util\t/abs/b/util/fixtures\ttestFixtures', 'project\t:app\tex\tapp\t1\tapp', - 'root\tr1\t:app\truntimeClasspath\t1', - 'node\tr1\tex:util:jar:1\tex\tutil\t1\tjar\t\t1\t:b:util', - 'node\tr1\tex:util:jar:test-fixtures:1\tex\tutil\t1\tjar\ttest-fixtures\t1\t:b:util', + 'root\tr1\t:app\truntimeClasspath\t1\tmain\truntime', + 'node\tr1\tex:util:jar:1\tex\tutil\t1\tjar\t\t1\t:b:util\tmain', + 'node\tr1\tex:util:jar:test-fixtures:1\tex\tutil\t1\tjar\ttest-fixtures\t1\t:b:util\ttestFixtures', 'node\tr1\tg:ext:jar:2\tg\text\t2\tjar\t\t0\t', 'edge\tr1\tex:util:jar:test-fixtures:1\tex:util:jar:1', 'edge\tr1\tex:util:jar:1\tg:ext:jar:2', - 'root\tr2\t:b:util\truntimeClasspath\t1', + 'root\tr2\t:b:util\truntimeClasspath\t1\tmain\truntime', 'node\tr2\tex:util:jar:1\tex\tutil\t1\tjar\t\t1\t:a:util', ].join('\n') const { artifactPaths, facts } = assembleFacts(parseRecords(records), { @@ -444,17 +473,24 @@ describe('records → assemble → sidecar', () => { ':app': [':b:util(g:ext:jar:2)'], ':b:util': [':a:util'], }) - expect(artifactPaths.classpathByProject.get(':app')).toEqual([ - ':b:util', - 'g:ext:jar:2', - ]) - expect(artifactPaths.classpathByProject.get(':b:util')).toEqual([':a:util']) - expect(artifactPaths.pathsById.get(':a:util')?.sources).toEqual([ - '/abs/a/util/src', + expect(artifactPaths.classpathsByProject.get(':app')?.get('main')).toEqual([ + { project: ':app', outputs: 'main' }, + { project: ':b:util', outputs: 'main' }, + { project: ':b:util', outputs: 'testFixtures' }, + { id: 'g:ext:jar:2' }, ]) - expect(artifactPaths.pathsById.get(':b:util')?.sources).toEqual([ - '/abs/b/util/src', + expect( + artifactPaths.classpathsByProject.get(':b:util')?.get('main'), + ).toEqual([ + { project: ':b:util', outputs: 'main' }, + { project: ':a:util', outputs: 'main' }, ]) + expect( + Object.fromEntries(artifactPaths.outputsByProject.get(':b:util') ?? []), + ).toEqual({ + main: { sources: ['/abs/b/util/src'], targets: [] }, + testFixtures: { sources: ['/abs/b/util/fixtures'], targets: [] }, + }) }) it('marks each subproject with its own build files and no component with any', () => { @@ -544,6 +580,138 @@ describe('records → assemble → sidecar', () => { }) }) +function classpaths( + records: string[], + project: string, +): Record { + return Object.fromEntries( + assembleFacts(parseRecords(records.join('\n')), { + fileExists: () => true, + }).artifactPaths.classpathsByProject.get(project) ?? [], + ) +} + +describe('classpaths', () => { + it("gives each output set its runtime classpath in the root's order, then compile-only entries", () => { + expect( + classpaths( + [ + 'meta\tgradle\t8.10\t17', + 'project\t:app\tg\tapp\t1\tapp', + 'outputSet\t:app\tmain\t', + 'outputSet\t:app\ttest\tmain', + 'outputSet\t:app\ttestFixtures\tmain', + 'root\t0\t:app\tannotationProcessor\t0', + 'node\t0\tg:proc:jar:1\tg\tproc\t1\tjar\t\t1', + 'root\t1\t:app\tcompileClasspath\t1\tmain\tcompile', + 'node\t1\tg:api:jar:1\tg\tapi\t1\tjar\t\t1', + 'node\t1\tg:servlet:jar:3\tg\tservlet\t3\tjar\t\t1', + 'root\t2\t:app\truntimeClasspath\t1\tmain\truntime', + 'node\t2\tg:z:jar:1\tg\tz\t1\tjar\t\t1', + 'node\t2\tg:api:jar:2\tg\tapi\t2\tjar\t\t1', + 'root\t3\t:app\ttestRuntimeClasspath\t0\ttest\truntime', + 'node\t3\tg:junit:jar:4\tg\tjunit\t4\tjar\t\t1', + 'node\t3\tg:z:jar:2\tg\tz\t2\tjar\t\t1', + ], + ':app', + ), + ).toEqual({ + main: [ + { project: ':app', outputs: 'main' }, + { id: 'g:z:jar:1' }, + { id: 'g:api:jar:2' }, + { id: 'g:servlet:jar:3' }, + ], + test: [ + { project: ':app', outputs: 'test' }, + { project: ':app', outputs: 'main' }, + { id: 'g:junit:jar:4' }, + { id: 'g:z:jar:2' }, + ], + testFixtures: [ + { project: ':app', outputs: 'testFixtures' }, + { project: ':app', outputs: 'main' }, + ], + }) + }) + + it("gives a set without a runtime classpath its own code, and adds no set it doesn't declare", () => { + expect( + classpaths( + [ + 'meta\tgradle\t8.10\t17', + 'project\t:app\tg\tapp\t1\tapp', + 'outputSet\t:app\tmain\t', + 'outputSet\t:app\tjvmTest\tjvm', + 'root\t0\t:app\tcompileClasspath\t1\tmain\tcompile', + 'node\t0\tg:api:jar:1\tg\tapi\t1\tjar\t\t1', + ], + ':app', + ), + ).toEqual({ + main: [{ project: ':app', outputs: 'main' }], + jvmTest: [ + { project: ':app', outputs: 'jvmTest' }, + { project: ':app', outputs: 'jvm' }, + ], + }) + }) + + it('takes classpaths the build tool states as they are, by component id', () => { + expect( + classpaths( + [ + 'meta\tmaven\t3.9.6\t17', + 'project\tg:app:1\tg\tapp\t1\tapp', + 'project\tg:lib:1\tg\tlib\t1\tlib', + 'outputSet\tg:app:1\tmain\t', + 'outputSet\tg:app:1\ttest\tmain', + 'root\t0\tg:app:1\tcompile\t1', + 'node\t0\tg:z:jar:1\tg\tz\t1\tjar\t\t1', + 'node\t0\tg:lib:1\tg\tlib\t1\t\t\t1\tg:lib:1', + 'root\t1\tg:app:1\ttest\t0', + 'node\t1\tg:bcpkix:jar:1\tg\tbcpkix\t1\tjar\t\t1', + 'classpath\tg:app:1\tmain\tg:lib:1\tmain', + 'classpath\tg:app:1\tmain\tg:z:jar:1\t', + 'classpath\tg:app:1\ttest\tg:bcpkix:jar:1\t', + 'classpath\tg:app:1\ttest\tg:lib:1\tmain,test', + 'classpath\tg:app:1\ttest\tg:z:jar:1\t', + ], + 'g:app:1', + ), + ).toEqual({ + main: [ + { project: 'g:app:1', outputs: 'main' }, + { project: 'g:lib:1', outputs: 'main' }, + { id: 'g:z:jar:1' }, + ], + test: [ + { project: 'g:app:1', outputs: 'test' }, + { project: 'g:app:1', outputs: 'main' }, + { id: 'g:bcpkix:jar:1' }, + { project: 'g:lib:1', outputs: 'main' }, + { project: 'g:lib:1', outputs: 'test' }, + { id: 'g:z:jar:1' }, + ], + }) + }) + + it('rejects a stated classpath entry that no root of the project resolved', () => { + expect(() => + classpaths( + [ + 'meta\tmaven\t3.9.6\t17', + 'project\tg:app:1\tg\tapp\t1\tapp', + 'root\t0\tg:app:1\tcompile\t1', + 'node\t0\tg:z:jar:1\tg\tz\t1\tjar\t\t1', + 'classpath\tg:app:1\tmain\tg:y:jar:1\t', + ], + 'g:app:1', + ), + ).toThrow(/names g:y:jar:1/) + }) +}) + describe('parseRecords', () => { it('reads the build root the build reports', () => { expect( diff --git a/src/commands/manifest/scripts/facts.mts b/src/commands/manifest/scripts/facts.mts index 6c48d539c..f74ba0d40 100644 --- a/src/commands/manifest/scripts/facts.mts +++ b/src/commands/manifest/scripts/facts.mts @@ -58,15 +58,27 @@ export type SocketFactsSbomProject = AnyPURL & { children: number[] } -// Resolved on-disk paths for a --with-files run, keyed by component or project -// id (a project and its own component share one). `targets` = classpath -// entries (jars / module output dirs), absent when the component's artifact -// did not resolve; `sources` = module source roots. +export type OutputPaths = { + sources: string[] + targets: string[] +} + +// One entry of a classpath, in the order the JVM searches it. +export type ClasspathEntry = + // An external artifact, by component id. + | { id: string } + // One output set (`main`, `test`, `testFixtures`, ...) of a build project. + | { project: string; outputs: string } + +// Resolved on-disk paths for a --with-files run, and each project's classpaths. export type ResolvedArtifactPaths = { - pathsById: Map - // Component ids on each project's resolved classpath (union over its - // configurations), keyed by project id. - classpathByProject: Map + // External artifacts by component id: `targets` are their files, absent when + // the artifact did not resolve. + pathsById: Map + // Build projects' output sets by name, keyed by project id. + outputsByProject: Map> + // Each project's classpaths by output set, keyed by project id. + classpathsByProject: Map> // Component ids of each project's direct dependencies, keyed by project id. directDependenciesByProject: Map } diff --git a/src/commands/manifest/scripts/maven-extension/src/main/java/tech/coana/socket/SocketFactsRecordsEngine.java b/src/commands/manifest/scripts/maven-extension/src/main/java/tech/coana/socket/SocketFactsRecordsEngine.java index 93c8ad272..b297c49c0 100644 --- a/src/commands/manifest/scripts/maven-extension/src/main/java/tech/coana/socket/SocketFactsRecordsEngine.java +++ b/src/commands/manifest/scripts/maven-extension/src/main/java/tech/coana/socket/SocketFactsRecordsEngine.java @@ -144,9 +144,18 @@ public void run(MavenSession session, List reactor, File rootDir, File pom = module.getFile(); if (pom != null && pom.isFile()) rec(lines, "projectBuild", key, SocketSupport.relativePath(rootDir.toPath(), pom.toPath())); } + // Maven puts a module's main classes on its test classpath. + if (!excluded || opts.withFiles) { + rec(lines, "outputSet", key, "main", ""); + rec(lines, "outputSet", key, "test", "main"); + } if (opts.withFiles) { - for (String s : collectSources(module)) rec(lines, "projectSrc", key, s); - for (String t : collectTargets(module)) rec(lines, "projectTgt", key, t); + for (String set : new String[] {"main", "test"}) { + boolean test = "test".equals(set); + for (String s : collectSources(module, test)) rec(lines, "projectSrc", key, s, set); + String target = test ? module.getBuild().getTestOutputDirectory() : module.getBuild().getOutputDirectory(); + if (target != null) rec(lines, "projectTgt", key, new File(target).getAbsolutePath(), set); + } } } @@ -160,11 +169,13 @@ public void run(MavenSession session, List reactor, File rootDir, // A wholly excluded reactor module is not resolved (matches the project-record skip above). if (SocketSupport.isExcludedDir(excludeBase, module.getBasedir().toPath(), excludes)) continue; Map nodes = new LinkedHashMap<>(); + ClasspathOrder order = new ClasspathOrder(); Set directIds = new HashSet<>(); // Which direct dependencies are prod-scoped: seeds the prod/dev root split (see emitModuleRoots). Set directProdIds = new HashSet<>(); - collectModule(session, module, passingScopes, reactorGavs, populateGavs, opts, nodes, directIds, directProdIds, failures, unscannable); + collectModule(session, module, passingScopes, reactorGavs, populateGavs, opts, nodes, order, directIds, directProdIds, failures, unscannable); rootIdx = emitModuleRoots(lines, rootIdx, projectKey(module), nodes, directIds, directProdIds); + emitClasspath(lines, projectKey(module), order); } for (Failure f : failures) rec(lines, "failure", f.coord, f.detail, f.config, f.project); @@ -189,6 +200,7 @@ private void collectModule( Set populateGavs, Options opts, Map nodes, + ClasspathOrder order, Set directIds, Set directProdIds, Set failures, @@ -299,7 +311,7 @@ private void collectModule( if (graph == null) return; Set visited = new HashSet<>(); for (DependencyNode child : graph.getChildren()) { - String id = visit(child, passingScopes, reactorGavs, unresolved, opts, nodes, visited); + String id = visit(child, passingScopes, reactorGavs, unresolved, opts, nodes, order, visited); if (id != null) { directIds.add(id); // A direct dependency's effective scope is its declared scope (Aether's scope selector keeps @@ -400,6 +412,7 @@ private String visit( Set unresolved, Options opts, Map nodes, + ClasspathOrder order, Set visited) { Dependency dep = dn.getDependency(); Artifact artifact = dn.getArtifact(); @@ -427,6 +440,7 @@ private String visit( ? upsert(nodes, id, artifact.getGroupId(), artifact.getArtifactId(), "", "", version, gav) : upsert(nodes, id, artifact.getGroupId(), artifact.getArtifactId(), type == null ? "" : type, classifier == null ? "" : classifier, version, ""); + order.add(id, scope, internal ? ("test-jar".equals(type) || "tests".equals(classifier) ? "test" : "main") : null); // Maven wrote each accepted node's resolved file back onto the node; a reactor module reports its // own dirs through its `project` record instead of a `file` record. if (!internal && opts.withFiles) { @@ -435,7 +449,7 @@ private String visit( } for (DependencyNode child : dn.getChildren()) { - String childId = visit(child, passingScopes, reactorGavs, unresolved, opts, nodes, visited); + String childId = visit(child, passingScopes, reactorGavs, unresolved, opts, nodes, order, visited); if (childId != null) node.children.add(childId); } return id; @@ -503,6 +517,19 @@ private int emitModuleRoots( return rootIdx; } + // Maven's pre-order walk of the module's graph (RepositoryUtils.toArtifacts), filtered by scope; `main` + // is the runtime classpath, then the compile-only (provided, system) entries. + private static void emitClasspath(List lines, String projectKey, ClasspathOrder order) { + emitClasspath(lines, projectKey, "main", order.main()); + emitClasspath(lines, projectKey, "test", order.test); + } + + private static void emitClasspath(List lines, String projectKey, String name, Map> entries) { + for (Map.Entry> e : entries.entrySet()) { + rec(lines, "classpath", projectKey, name, e.getKey(), String.join(",", e.getValue())); + } + } + // Every node id reachable from `seeds` by following retained child edges. private static Set reachable(Map nodes, Set seeds) { Set seen = new HashSet<>(); @@ -577,29 +604,19 @@ private Set readPopulateGavs(Options opts) throws IOException { // Configured source roots, emitted unconditionally (like gradle's srcDirs / sbt's sourceDirectories): // the analysis never builds the project, so these need not exist yet. - private List collectSources(MavenProject module) { + private List collectSources(MavenProject module, boolean test) { Set sources = new TreeSet<>(); - addPaths(sources, module.getCompileSourceRoots()); - addPaths(sources, module.getTestCompileSourceRoots()); - for (Resource r : module.getBuild().getResources()) addPath(sources, r.getDirectory()); - for (Resource r : module.getBuild().getTestResources()) addPath(sources, r.getDirectory()); + addPaths(sources, test ? module.getTestCompileSourceRoots() : module.getCompileSourceRoots()); + for (Resource r : test ? module.getBuild().getTestResources() : module.getBuild().getResources()) { + addPath(sources, r.getDirectory()); + } // generated-source roots aren't on the model without a build; best-effort pick up the // conventional dirs only if a prior `mvn compile` already produced them (else they'd be guesses). String buildDir = module.getBuild().getDirectory(); - addExisting(sources, buildDir + File.separator + "generated-sources"); - addExisting(sources, buildDir + File.separator + "generated-test-sources"); + addExisting(sources, buildDir + File.separator + (test ? "generated-test-sources" : "generated-sources")); return new ArrayList<>(sources); } - // Configured compiled-output dirs, emitted unconditionally (like gradle's classesDirs / sbt's - // classDirectory): the analysis never builds the project, so these need not exist yet. - private List collectTargets(MavenProject module) { - Set targets = new TreeSet<>(); - addPath(targets, module.getBuild().getOutputDirectory()); - addPath(targets, module.getBuild().getTestOutputDirectory()); - return new ArrayList<>(targets); - } - private static void addPaths(Set acc, List dirs) { if (dirs == null) return; for (String d : dirs) addPath(acc, d); @@ -873,6 +890,37 @@ public int hashCode() { } } + // Node ids in walk order by classpath, each with the output sets of a reactor module its artifacts bring in + // (its jar main, its test-jar test). + private static final class ClasspathOrder { + final Map> runtime = new LinkedHashMap<>(); + final Map> compileOnly = new LinkedHashMap<>(); + final Map> test = new LinkedHashMap<>(); + + void add(String id, String scope, String outputSet) { + if ("compile".equals(scope) || "runtime".equals(scope)) put(runtime, id, outputSet); + else if (!"test".equals(scope)) put(compileOnly, id, outputSet); + put(test, id, outputSet); + } + + // The runtime classpath, then the compile-only entries. + Map> main() { + Map> main = new LinkedHashMap<>(); + for (Map> part : Arrays.asList(runtime, compileOnly)) { + part.forEach((id, sets) -> { + put(main, id, null); + main.get(id).addAll(sets); + }); + } + return main; + } + + private static void put(Map> entries, String id, String outputSet) { + Set sets = entries.computeIfAbsent(id, k -> new LinkedHashSet<>()); + if (outputSet != null) sets.add(outputSet); + } + } + private static final class Node { final String id; final String groupId; diff --git a/src/commands/manifest/scripts/records.mts b/src/commands/manifest/scripts/records.mts index 2e99cd630..69c8d66eb 100644 --- a/src/commands/manifest/scripts/records.mts +++ b/src/commands/manifest/scripts/records.mts @@ -1,3 +1,4 @@ +import type { OutputPaths } from './facts.mts' import type { ResolutionFailure, Unscannable } from './resolution-report.mts' // Line-protocol the build-tool scripts emit to a records file (NOT stdout — sbt @@ -10,13 +11,15 @@ import type { ResolutionFailure, Unscannable } from './resolution-report.mts' // buildRoot path (absolute; the facts file's directory) // entry path (file-addressed builds; build-root-relative) // project projectKey group name version dir -// projectSrc projectKey path (--with-files only) -// projectTgt projectKey path (--with-files only) +// projectSrc projectKey path outputSet (--with-files only) +// projectTgt projectKey path outputSet (--with-files only) +// outputSet projectKey outputSet includes (the project's own sets on its classpath) // projectBuild projectKey path (build-root-relative; may be absent on disk) // excludedProject projectKey (--with-files only; paths without a project) -// root rootId projectKey config prod(0|1) -// node rootId coordId group name version ext classifier direct(0|1) project +// root rootId projectKey config prod(0|1) outputSet role(runtime|compile) +// node rootId coordId group name version ext classifier direct(0|1) project outputSets(brought in) // edge rootId parentCoordId childCoordId +// classpath projectKey outputSet coordId outputSets (in order; see below) // file rootId coordId path (--with-files only) // unfetched rootId coordId (--with-files only; requested artifact not fetched) // scanned config @@ -29,6 +32,10 @@ import type { ResolutionFailure, Unscannable } from './resolution-report.mts' // A `root` is one (subproject, configuration) resolution root; `coordId` is the // coordinate key (`group:name:ext:classifier:version`, empty segments dropped), // used opaquely as the per-root node key. Unknown tags are ignored. +// +// Output sets are a project's code by source set (`main`, `test`, ...). A root +// with one is that set's runtime or compile classpath, its nodes in classpath +// order; `classpath` records state Maven's outright. export type RawCoord = { group: string @@ -45,6 +52,8 @@ export type RawNode = { // projectKey of the build project this node resolves to; empty for an // external artifact. project: string + // The build project's output sets this dependency brings in. + outputSets: string[] // --with-files only. targets: string[] unfetched: boolean @@ -53,8 +62,10 @@ export type RawNode = { export type RawRoot = { rootId: string projectKey: string - config: string prod: boolean + // The output set whose classpath this root is, or '' for none. + outputSet: string + role: 'runtime' | 'compile' | '' nodes: Map edges: Array<[string, string]> } @@ -65,12 +76,18 @@ export type RawProject = { name: string version: string dir: string - sources: string[] - targets: string[] + outputs: Map + // Each output set, with the project's other output sets on its classpath. + outputSets: Map // The project's build files, build-root-relative. buildFiles: string[] } +export type StatedClasspathEntry = { + coordId: string + outputSets: string[] +} + export type ParsedRecords = { tool: string toolVersion: string @@ -85,6 +102,8 @@ export type ParsedRecords = { // the kept projects depending on them. excludedProjects: Map roots: Map + // By projectKey, then output set: nodes of the project's roots, in order. + classpaths: Map> scannedConfigs: string[] failures: ResolutionFailure[] unscannable: Unscannable[] @@ -111,6 +130,14 @@ function bool(s: string | undefined): boolean { return s === '1' || s === 'true' } +function list(s: string | undefined): string[] { + return s ? s.split(',') : [] +} + +function role(s: string | undefined): RawRoot['role'] { + return s === 'runtime' || s === 'compile' ? s : '' +} + export function parseRecords(text: string): ParsedRecords { const result: ParsedRecords = { tool: '', @@ -121,6 +148,7 @@ export function parseRecords(text: string): ParsedRecords { projects: new Map(), excludedProjects: new Map(), roots: new Map(), + classpaths: new Map(), scannedConfigs: [], failures: [], unscannable: [], @@ -134,8 +162,9 @@ export function parseRecords(text: string): ParsedRecords { r = { rootId: id, projectKey: '', - config: '', prod: false, + outputSet: '', + role: '', nodes: new Map(), edges: [], } @@ -152,8 +181,8 @@ export function parseRecords(text: string): ParsedRecords { name: '', version: '', dir: '', - sources: [], - targets: [], + outputs: new Map(), + outputSets: new Map(), buildFiles: [], } result.projects.set(key, p) @@ -161,6 +190,16 @@ export function parseRecords(text: string): ParsedRecords { return p } + const outputs = (key: string, set: string): OutputPaths => { + const all = project(key).outputs + let o = all.get(set) + if (!o) { + o = { sources: [], targets: [] } + all.set(set, o) + } + return o + } + for (const rawLine of text.split('\n')) { if (!rawLine) { continue @@ -187,13 +226,18 @@ export function parseRecords(text: string): ParsedRecords { break } case 'projectSrc': - if (f[2]) { - project(f[1] ?? '').sources.push(f[2]) + if (f[2] && f[3]) { + outputs(f[1] ?? '', f[3]).sources.push(f[2]) } break case 'projectTgt': + if (f[2] && f[3]) { + outputs(f[1] ?? '', f[3]).targets.push(f[2]) + } + break + case 'outputSet': if (f[2]) { - project(f[1] ?? '').targets.push(f[2]) + project(f[1] ?? '').outputSets.set(f[2], list(f[3])) } break case 'excludedProject': @@ -207,8 +251,9 @@ export function parseRecords(text: string): ParsedRecords { case 'root': { const r = root(f[1] ?? '') r.projectKey = f[2] ?? '' - r.config = f[3] ?? '' r.prod = bool(f[4]) + r.outputSet = f[5] ?? '' + r.role = role(f[6]) break } case 'node': { @@ -225,6 +270,7 @@ export function parseRecords(text: string): ParsedRecords { }, direct: bool(f[8]), project: f[9] ?? '', + outputSets: list(f[10]), targets: [], unfetched: false, }) @@ -238,6 +284,24 @@ export function parseRecords(text: string): ParsedRecords { } break } + case 'classpath': { + const key = f[1] ?? '' + let byName = result.classpaths.get(key) + if (!byName) { + byName = new Map() + result.classpaths.set(key, byName) + } + const name = f[2] ?? '' + let cp = byName.get(name) + if (!cp) { + cp = [] + byName.set(name, cp) + } + if (f[3]) { + cp.push({ coordId: f[3], outputSets: list(f[4]) }) + } + break + } case 'unfetched': { const node = root(f[1] ?? '').nodes.get(f[2] ?? '') if (node) { diff --git a/src/commands/manifest/scripts/sidecar.mts b/src/commands/manifest/scripts/sidecar.mts index a19b54212..e98a04815 100644 --- a/src/commands/manifest/scripts/sidecar.mts +++ b/src/commands/manifest/scripts/sidecar.mts @@ -1,29 +1,32 @@ import type { AnyPURL, + ClasspathEntry, + OutputPaths, ResolvedArtifactPaths, SocketFactsSbom, SocketFactsSbomComponent, SocketFactsSbomProject, } from './facts.mts' +// A build project's code by output set (`main`, `test`, `testFixtures`, ...). +export type SidecarOutputs = Record + export type SidecarComponentEntry = SocketFactsSbomComponent & { - // Classpath entries (jars, or a sibling project's own build output dirs - // when this component is that project). `[]` means resolved and found - // nothing (e.g. a pom/BOM); undefined means paths were not resolved. + // An external artifact's files: `[]` resolved and found nothing (a pom/BOM), + // undefined not resolved. `sources` is always `[]`. targets?: string[] | undefined - // First-party source roots; `[]` for a genuinely external dependency (still - // attempted, nothing to find), not undefined. sources?: string[] | undefined + // A sibling build project's output sets, in place of `targets`/`sources`. + outputs?: SidecarOutputs | undefined } export type SidecarProjectEntry = Omit & { - targets?: string[] | undefined - sources?: string[] | undefined + outputs?: SidecarOutputs | undefined // Ids of this facts file's components[] the project depends on directly. dependencies: string[] - // Ids of this facts file's components[] forming the project's full - // transitive classpath across all its configurations. - classpath: string[] + // The runtime classpath of each output set, the project's own code first. + // When artifact paths were resolved, in the order the JVM searches them. + classpaths: Record } // Frozen contract with `coana run --compute-artifacts-sidecar`; change only @@ -42,8 +45,8 @@ export type ResolvedPathsSidecar = Record< projects: SidecarProjectEntry[] // This reactor's dependency-position entries: genuinely external // artifacts, and dependency edges that resolve to a sibling first-party - // project (reported via that project's own source/target roots instead - // of a jar path). + // project (reported via that project's output sets instead of a jar + // path). components: SidecarComponentEntry[] } > @@ -53,18 +56,26 @@ export type SidecarAccumulator = Map< { projects: SidecarProjectEntry[]; components: SidecarComponentEntry[] } > -// `[]` means resolved and found nothing (e.g. a pom/BOM with no artifact); no -// `targets` means the artifact did not resolve. +// A build project carries its output sets; an external artifact its files, +// `targets` absent when it did not resolve. function attachPaths( entry: T, artifactPaths: ResolvedArtifactPaths, -): T & { targets?: string[]; sources: string[] } { +): T & { + outputs?: SidecarOutputs + sources?: string[] + targets?: string[] +} { + const outputs = artifactPaths.outputsByProject.get(entry.id) + if (outputs) { + return { ...entry, outputs: Object.fromEntries(outputs) } + } const paths = artifactPaths.pathsById.get(entry.id) const targets = paths ? paths.targets : [] return { ...entry, ...(targets ? { targets: [...targets] } : {}), - sources: [...(paths?.sources ?? [])], + sources: [], } } @@ -103,7 +114,9 @@ export function accumulateSidecar( dependencies: [ ...(artifactPaths.directDependenciesByProject.get(proj.id) ?? []), ], - classpath: [...(artifactPaths.classpathByProject.get(proj.id) ?? [])], + classpaths: Object.fromEntries( + artifactPaths.classpathsByProject.get(proj.id) ?? [], + ), }), ), }) diff --git a/src/commands/manifest/scripts/sidecar.test.mts b/src/commands/manifest/scripts/sidecar.test.mts index 412819d19..53cedb773 100644 --- a/src/commands/manifest/scripts/sidecar.test.mts +++ b/src/commands/manifest/scripts/sidecar.test.mts @@ -14,7 +14,8 @@ import type { SidecarAccumulator } from './sidecar.mts' function emptyArtifactPaths(): ResolvedArtifactPaths { return { pathsById: new Map(), - classpathByProject: new Map(), + outputsByProject: new Map(), + classpathsByProject: new Map(), directDependenciesByProject: new Map(), } } @@ -24,7 +25,7 @@ function mkComponentFixture(target: string): { paths: ResolvedArtifactPaths } { const paths = emptyArtifactPaths() - paths.pathsById.set('g:a:jar:1', { sources: [], targets: [target] }) + paths.pathsById.set('g:a:jar:1', { targets: [target] }) return { facts: { components: [ @@ -67,7 +68,10 @@ describe('compute-artifacts sidecar', () => { ], } const artifactPaths = emptyArtifactPaths() - artifactPaths.classpathByProject.set(':app', ['g:a:jar:1']) + artifactPaths.classpathsByProject.set( + ':app', + new Map([['main', [{ id: 'g:a:jar:1' }]]]), + ) artifactPaths.directDependenciesByProject.set(':app', ['g:a:jar:1']) const acc: SidecarAccumulator = new Map() @@ -87,12 +91,12 @@ describe('compute-artifacts sidecar', () => { name: 'app', subprojectDir: 'app', dependencies: ['g:a:jar:1'], - classpath: ['g:a:jar:1'], + classpaths: { main: [{ id: 'g:a:jar:1' }] }, }) expect(entry.components[0]).toEqual(facts.components[0]) }) - it('carries a component through with resolved targets/sources attached, keyed by its own facts file', () => { + it('carries a component through with resolved targets attached, keyed by its own facts file', () => { const facts: SocketFactsSbom = { components: [ { @@ -107,7 +111,6 @@ describe('compute-artifacts sidecar', () => { } const artifactPaths = emptyArtifactPaths() artifactPaths.pathsById.set('com.example:lib:jar:da517db', { - sources: ['/abs/lib/src/main/java'], targets: ['/abs/lib.jar'], }) @@ -127,7 +130,7 @@ describe('compute-artifacts sidecar', () => { qualifiers: { ext: 'jar' }, id: 'com.example:lib:jar:da517db', targets: ['/abs/lib.jar'], - sources: ['/abs/lib/src/main/java'], + sources: [], }, ], }, @@ -161,7 +164,7 @@ describe('compute-artifacts sidecar', () => { expect(entry.sources).toEqual([]) }) - it("gives each sibling project's component that project's own paths, even when they share a coordinate", () => { + it("gives each sibling project's component that project's own output sets, even when they share a coordinate", () => { const util = { type: 'maven', namespace: 'ex', @@ -180,14 +183,24 @@ describe('compute-artifacts sidecar', () => { ], } const artifactPaths = emptyArtifactPaths() - artifactPaths.pathsById.set(':a:util', { - sources: ['/abs/a/util/src'], - targets: ['/abs/a/util/classes'], - }) - artifactPaths.pathsById.set(':b:util', { - sources: ['/abs/b/util/src'], - targets: ['/abs/b/util/classes'], - }) + artifactPaths.outputsByProject.set( + ':a:util', + new Map([ + [ + 'main', + { sources: ['/abs/a/util/src'], targets: ['/abs/a/util/classes'] }, + ], + ]), + ) + artifactPaths.outputsByProject.set( + ':b:util', + new Map([ + [ + 'main', + { sources: ['/abs/b/util/src'], targets: ['/abs/b/util/classes'] }, + ], + ]), + ) const acc: SidecarAccumulator = new Map() accumulateSidecar( @@ -199,7 +212,11 @@ describe('compute-artifacts sidecar', () => { const entry = serializeSidecar(acc)['/root/gradle.socket.facts.json']! for (const entries of [entry.components, entry.projects]) { - expect(Object.fromEntries(entries.map(e => [e.id, e.sources]))).toEqual({ + expect( + Object.fromEntries( + entries.map(e => [e.id, e.outputs?.['main']?.sources]), + ), + ).toEqual({ ':a:util': ['/abs/a/util/src'], ':b:util': ['/abs/b/util/src'], }) @@ -234,7 +251,7 @@ describe('compute-artifacts sidecar', () => { expect(entry.direct).toBe(true) }) - it('carries a first-party module (project, not a component) source/target roots, keyed by its own facts file', () => { + it('carries a first-party module (project, not a component) output sets, keyed by its own facts file', () => { const facts: SocketFactsSbom = { // The app module is a project but nothing depends on it, so it is absent // from components — its source roots must still reach the sidecar. @@ -252,10 +269,25 @@ describe('compute-artifacts sidecar', () => { ], } const artifactPaths = emptyArtifactPaths() - artifactPaths.pathsById.set('com.example:app:1.0', { - sources: ['/abs/app/src/main/java'], - targets: ['/abs/app/build/classes'], - }) + artifactPaths.outputsByProject.set( + 'com.example:app:1.0', + new Map([ + [ + 'main', + { + sources: ['/abs/app/src/main/java'], + targets: ['/abs/app/classes'], + }, + ], + [ + 'test', + { + sources: ['/abs/app/src/test/java'], + targets: ['/abs/app/test-classes'], + }, + ], + ]), + ) const acc: SidecarAccumulator = new Map() accumulateSidecar(acc, facts, artifactPaths, '/root/app/.socket.facts.json') @@ -271,14 +303,22 @@ describe('compute-artifacts sidecar', () => { version: '1.0', subprojectDir: 'app', dependencies: [], - targets: ['/abs/app/build/classes'], - sources: ['/abs/app/src/main/java'], - classpath: [], + outputs: { + main: { + sources: ['/abs/app/src/main/java'], + targets: ['/abs/app/classes'], + }, + test: { + sources: ['/abs/app/src/test/java'], + targets: ['/abs/app/test-classes'], + }, + }, + classpaths: {}, }, ]) }) - it('attaches each project its own classpath ids, keyed by project id even within one directory', () => { + it('attaches each project its own classpaths, keyed by project id even within one directory', () => { const project = { type: 'maven', namespace: 'com.example', @@ -294,16 +334,22 @@ describe('compute-artifacts sidecar', () => { ], } const artifactPaths = emptyArtifactPaths() - artifactPaths.classpathByProject.set('x/a.xml', ['g:x:jar:1']) - artifactPaths.classpathByProject.set('x/b.xml', ['g:x:jar:2']) + artifactPaths.classpathsByProject.set( + 'x/a.xml', + new Map([['main', [{ id: 'g:x:jar:1' }]]]), + ) + artifactPaths.classpathsByProject.set( + 'x/b.xml', + new Map([['main', [{ id: 'g:x:jar:2' }]]]), + ) const acc: SidecarAccumulator = new Map() accumulateSidecar(acc, facts, artifactPaths, '/root/.socket.facts.json') const projects = serializeSidecar(acc)['/root/.socket.facts.json']!.projects - expect(projects.map(p => p.classpath)).toEqual([ - ['g:x:jar:1'], - ['g:x:jar:2'], + expect(projects.map(p => p.classpaths)).toEqual([ + { main: [{ id: 'g:x:jar:1' }] }, + { main: [{ id: 'g:x:jar:2' }] }, ]) }) @@ -339,15 +385,15 @@ describe('compute-artifacts sidecar', () => { ], } const pathsA = emptyArtifactPaths() - pathsA.pathsById.set('com.example:shared:1.0', { - sources: ['/root-a/src/main/java'], - targets: [], - }) + pathsA.outputsByProject.set( + 'com.example:shared:1.0', + new Map([['main', { sources: ['/root-a/src/main/java'], targets: [] }]]), + ) const pathsB = emptyArtifactPaths() - pathsB.pathsById.set('com.example:shared:1.0', { - sources: ['/root-b/src/main/java'], - targets: [], - }) + pathsB.outputsByProject.set( + 'com.example:shared:1.0', + new Map([['main', { sources: ['/root-b/src/main/java'], targets: [] }]]), + ) const acc: SidecarAccumulator = new Map() accumulateSidecar( @@ -369,10 +415,12 @@ describe('compute-artifacts sidecar', () => { '/root-b/.socket.facts.json', ]) expect( - resolved['/root-a/.socket.facts.json']!.projects[0]!.sources, + resolved['/root-a/.socket.facts.json']!.projects[0]!.outputs?.['main'] + ?.sources, ).toEqual(['/root-a/src/main/java']) expect( - resolved['/root-b/.socket.facts.json']!.projects[0]!.sources, + resolved['/root-b/.socket.facts.json']!.projects[0]!.outputs?.['main'] + ?.sources, ).toEqual(['/root-b/src/main/java']) }) diff --git a/src/commands/manifest/scripts/socket-facts.init.gradle b/src/commands/manifest/scripts/socket-facts.init.gradle index 7558a68c6..8d131a930 100644 --- a/src/commands/manifest/scripts/socket-facts.init.gradle +++ b/src/commands/manifest/scripts/socket-facts.init.gradle @@ -10,6 +10,12 @@ import java.util.Collections // `Project.findProperty` only exists since Gradle 2.13; fall back to hasProperty/property for older Gradle. gradle.ext.socketProp = { proj, name -> proj.hasProperty(name) ? proj.property(name) : null } +// String.uncapitalize() needs Groovy 2.4.8; Gradle 1.12 bundles Groovy 1.8. +gradle.ext.socketDecap = { String n -> n ? n[0].toLowerCase() + n.substring(1) : n } +// The first of the named properties `obj` has, or null. +gradle.ext.socketFirstProp = { obj, List names -> + names.findResult { n -> try { obj."$n" } catch (Exception ignore) { null } } +} // `-Psocket.excludePaths` → PRE-COMPILED, comma-joined, anchored regex pattern sources, used only // to skip whole excluded subprojects (source-file-level exclusion is left to the reachability @@ -79,6 +85,11 @@ gradle.ext.socketFactsState = [ projectGaByPath : Collections.synchronizedMap([:]), // Project path -> "group:name:version", the coordinate a failed project dependency is reported by. projectGavByPath : Collections.synchronizedMap([:]), + // Project path -> output set -> the project's other output sets its classpath holds. + projectOutputSets : Collections.synchronizedMap([:]), + // Project path -> configuration name -> [set:, role: runtime|compile]: the output set whose classpath + // the configuration resolves. + classpathRoles : Collections.synchronizedMap([:]), ] // Capture every project's identity before collectors run so they can recognize intra-project deps @@ -115,75 +126,166 @@ gradle.projectsEvaluated { g -> // ALL source sets are emitted (not just main/test) so custom ones (integrationTest, // functionalTest, ...) reach the analysis; source-file-level exclusion is coana's job. // `classesDirs` (Gradle 4+) falls back to the legacy single `classesDir` on older Gradle. - def sources = [] as Set - def targets = [] as Set + // Output set -> its source and class dirs (--with-files only) and the project's other output sets its + // classpath holds. + def outputs = [:] + def output = { String set -> outputs.get(set, [sources: [] as Set, targets: [] as Set, includes: []]) } + def withFiles = g.ext.socketProp.call(p, 'socket.withFiles')?.toString()?.toLowerCase() == 'true' + def addDirs = { String set, String kind, dirs -> if (withFiles) output(set)[kind].addAll(dirs*.absolutePath) } + def roles = [:] + def classpathOf = { String set, String runtimeName, String compileName -> + output(set) + if (runtimeName) roles[runtimeName] = [set: set, role: 'runtime'] + if (compileName) roles[compileName] = [set: set, role: 'compile'] + } + def holds = { String set, List others -> + def o = output(set) + o.includes = (o.includes + others.findAll { it != null && it != set }).unique() + } + // The output sets wired into a classpath file collection directly (test's holds main's output). + def wiredOutputs = { fc -> + def found = [] + def seen = [] as Set + def walk + walk = { x -> + if (x == null || !seen.add(System.identityHashCode(x))) return + if (x instanceof org.gradle.api.tasks.SourceSetOutput) { found << x; return } + if (x instanceof Collection || x instanceof Object[]) { x.each { walk(it) }; return } + try { if (x.metaClass.respondsTo(x, 'getFrom')) x.from.each { walk(it) } } catch (Exception ignore) {} + } + walk(fc) + found + } try { if (p.hasProperty('sourceSets')) { p.sourceSets.each { ss -> if (ss != null) { - ss.allSource.srcDirs.each { d -> sources << d.absolutePath } + classpathOf(ss.name, + g.ext.socketFirstProp.call(ss, ['runtimeClasspathConfigurationName', 'runtimeConfigurationName']), + g.ext.socketFirstProp.call(ss, ['compileClasspathConfigurationName', 'compileConfigurationName'])) + addDirs(ss.name, 'sources', ss.allSource.srcDirs) try { - ss.output.classesDirs.each { d -> targets << d.absolutePath } + addDirs(ss.name, 'targets', ss.output.classesDirs) } catch (Exception e1) { - try { targets << ss.output.classesDir.absolutePath } catch (Exception ignore) {} + try { addDirs(ss.name, 'targets', [ss.output.classesDir]) } catch (Exception ignore) {} } } } + p.sourceSets.each { ss -> + try { + holds(ss.name, wiredOutputs(ss.runtimeClasspath).collect { out -> p.sourceSets.find { it.output.is(out) }?.name }) + } catch (Exception ignore) {} + } } } catch (Exception ignore) {} // Kotlin compilations for the JVM (kotlin-jvm, KMP `jvm()`, Android): Kotlin source sets, shared // ones like commonMain included, are not java source sets. Untyped: KGP is not on this classpath. + // A compilation takes its java source set's output set (kotlin-jvm), its variant's (Android) or its + // target and name (KMP: jvm, jvmTest), and holds its associated compilations (jvmTest: jvm). def kotlin = p.extensions.findByName('kotlin') if (kotlin != null) { try { + def setOf = { t, c, String runtimeName -> + roles[runtimeName]?.set ?: (t.platformType.name() == 'androidJvm' || !t.name ? c.name : + (c.name == 'main' ? t.name : t.name + c.name.capitalize())) + } (kotlin.hasProperty('targets') ? kotlin.targets : [kotlin.target]).each { t -> try { if (t.platformType.name() in ['jvm', 'androidJvm']) { t.compilations.each { c -> + def runtimeName = g.ext.socketFirstProp.call(c, ['runtimeDependencyConfigurationName']) + def set = setOf(t, c, runtimeName) + classpathOf(set, runtimeName, g.ext.socketFirstProp.call(c, ['compileDependencyConfigurationName'])) c.allKotlinSourceSets.each { ks -> - sources.addAll(ks.kotlin.srcDirs*.absolutePath) - sources.addAll(ks.resources.srcDirs*.absolutePath) + addDirs(set, 'sources', ks.kotlin.srcDirs) + addDirs(set, 'sources', ks.resources.srcDirs) } - targets.addAll(c.output.classesDirs*.absolutePath) + addDirs(set, 'targets', c.output.classesDirs) + def associated = g.ext.socketFirstProp.call(c, ['allAssociatedCompilations', 'associatedCompilations', 'associateWith']) + holds(set, ((associated ?: []) as List).collect { a -> setOf(t, a, null) }) } } } catch (Exception ignore) {} } } catch (Exception ignore) {} } - // AGP keeps its source sets in the `android` extension and leaves the java ones empty. + // AGP keeps its source sets in the `android` extension and leaves the java ones empty. Output sets + // are variants (debug, debugUnitTest, debugAndroidTest); a test variant holds the variant it tests. def android = p.extensions.findByName('android') if (android != null) { - try { - android.sourceSets.each { ss -> + def destinationOf = { task -> + g.ext.socketFirstProp.call(task, ['destinationDirectory'])?.get()?.asFile ?: g.ext.socketFirstProp.call(task, ['destinationDir']) + } + def addSources = { String set, provider -> + ['javaDirectories', 'resourcesDirectories', 'kotlinDirectories'].each { prop -> + try { addDirs(set, 'sources', provider."$prop") } catch (Exception ignore) {} + } + } + // The variant API (AGP before 9) names each variant's source sets, classpaths, compile task and + // tested variant. Kotlin class dirs come from the compilations above. + def variants = [] + ['applicationVariants', 'libraryVariants', 'testVariants', 'unitTestVariants'].each { prop -> + try { if (android.hasProperty(prop)) variants.addAll(android."$prop") } catch (Exception ignore) {} + } + if (variants) { + variants.each { v -> try { - sources.addAll(ss.java.srcDirs*.absolutePath) - sources.addAll(ss.resources.srcDirs*.absolutePath) - // Last: older AGP has no `kotlin`. - sources.addAll(ss.kotlin.srcDirs*.absolutePath) + classpathOf(v.name, v.runtimeConfiguration?.name, v.compileConfiguration?.name) + v.sourceSets.each { sp -> addSources(v.name, sp) } + def compile = g.ext.socketFirstProp.call(v, ['javaCompileProvider'])?.get() ?: g.ext.socketFirstProp.call(v, ['javaCompile']) + if (compile != null) addDirs(v.name, 'targets', [destinationOf(compile)].findAll()) + holds(v.name, [g.ext.socketFirstProp.call(v, ['testedVariant'])?.name]) } catch (Exception ignore) {} } - } catch (Exception ignore) {} - // Class dirs come from the compile tasks: AGP's built-in Kotlin (AGP 9) leaves the Kotlin - // compilations' classesDirs empty. - def compileTypes = [JavaCompile] - if (kotlin != null) { - try { - compileTypes << Class.forName('org.jetbrains.kotlin.gradle.tasks.KotlinCompileTool', false, kotlin.getClass().classLoader) - } catch (Throwable ignore) {} - } - compileTypes.each { type -> - try { - p.tasks.withType(type).each { t -> - try { - targets << t.destinationDirectory.get().asFile.absolutePath - } catch (Exception e1) { - try { targets << t.destinationDir.absolutePath } catch (Exception ignore) {} + } else { + // AGP 9 has no variant API, so AGP's naming: class dirs from the compile tasks (its built-in Kotlin + // leaves classesDirs empty), source sets to the variants of their kind whose name has their words. + def compileTypes = [JavaCompile] + if (kotlin != null) { + try { + compileTypes << Class.forName('org.jetbrains.kotlin.gradle.tasks.KotlinCompileTool', false, kotlin.getClass().classLoader) + } catch (Throwable ignore) {} + } + def androidSets = [] as LinkedHashSet + compileTypes.each { type -> + try { + p.tasks.withType(type).each { t -> + def m = t.name =~ /^compile(.+?)(JavaWithJavac|Kotlin)$/ + if (!m.matches()) return + def set = g.ext.socketDecap.call(m.group(1)) + androidSets << set + classpathOf(set, set + 'RuntimeClasspath', set + 'CompileClasspath') + try { addDirs(set, 'targets', [destinationOf(t)].findAll()) } catch (Exception ignore) {} } - } - } catch (Exception ignore) {} + } catch (Exception ignore) {} + } + def kinds = [androidTest: 'AndroidTest', testFixtures: 'TestFixtures', test: 'UnitTest'] + androidSets.each { set -> + def kind = kinds.values().find { set.endsWith(it) && set.length() > it.length() } + def tested = kind ? set.substring(0, set.length() - kind.length()) : null + if (androidSets.contains(tested)) holds(set, [tested]) + } + if (withFiles) { + def words = { String n -> (n =~ /[A-Z]?[a-z0-9]+/).collect { it.toLowerCase() } } + def setWords = androidSets.collectEntries { [it, words(it)] } + try { + android.sourceSets.each { ss -> + def kind = kinds.find { k, v -> ss.name.startsWith(k) } + def scope = words(kind ? ss.name.substring(kind.key.length()) : (ss.name == 'main' ? '' : ss.name)) + androidSets.findAll { v -> + (kind ? v.endsWith(kind.value) : !kinds.values().any { v.endsWith(it) }) && setWords[v].containsAll(scope) + }.each { set -> + ['java', 'resources', 'kotlin'].each { prop -> + try { addDirs(set, 'sources', ss."$prop".srcDirs) } catch (Exception ignore) {} + } + } + } + } catch (Exception ignore) {} + } } } + g.socketFactsState.projectOutputSets[p.path] = outputs.collectEntries { set, o -> [set, o.includes] } + g.socketFactsState.classpathRoles[p.path] = roles // `buildFile` is the configured script, absent on disk for a project configured from the root or // a convention plugin. Still emitted: a direct dependency's mark must name its subproject. def buildFiles = [] @@ -197,8 +299,9 @@ gradle.projectsEvaluated { g -> name : p.name, version: (p.version ?: '').toString(), dir : rel(p.projectDir), - sources: (sources as List).sort(), - targets: (targets as List).sort(), + outputs: outputs.collectEntries { set, o -> + [set, [sources: (o.sources as List).sort(), targets: (o.targets as List).sort(), includes: o.includes]] + }, buildFiles: buildFiles, excluded: excluded, ]) @@ -283,14 +386,25 @@ allprojects { project -> } // A node is created once; its prod flag accumulates (OR) across the configs that reach it. - def upsertNode = { Map coord, boolean isProd, String projPath -> + // The output set a dependency on a build project brings in, by the variant it selected: + // `[]RuntimeElements`/`ApiElements`, or `default` and the like for the main one. + def outputSetOf = { String projPath, String targetConfig -> + def m = (targetConfig ?: '') =~ /^(.+?)(RuntimeElements|ApiElements)$/ + def set = m.matches() ? gradle.ext.socketDecap.call(m.group(1)) : 'main' + state.projectOutputSets[projPath]?.containsKey(set) ? set : 'main' + } + + def upsertNode = { Map coord, boolean isProd, String projPath, String outputSet = null -> def id = coordId(coord) synchronized (nodes) { def node = nodes[id] if (node == null) { - node = [coord: coord, children: [] as Set, prod: false, project: projPath] + node = [coord: coord, children: [] as Set, prod: false, project: projPath, outputSets: [] as LinkedHashSet] nodes[id] = node } + if (outputSet != null) { + node.outputSets << outputSet + } if (isProd) { node.prod = true } @@ -337,6 +451,7 @@ allprojects { project -> def gav = "${dep.moduleGroup ?: ''}:${dep.moduleName}:${dep.moduleVersion ?: ''}".toString() def shouldMaterialize = withFiles && (populateGavs == null || populateGavs.contains(gav)) def projPath = projectByGav[gav] + def outputSet = projPath == null ? null : outputSetOf(projPath, dep.configuration) def artifacts def artifactsError = null try { @@ -360,7 +475,7 @@ allprojects { project -> version : dep.moduleVersion ?: '', classifier: '', ext : ext, - ], isProd, projPath) + ], isProd, projPath, outputSet) } } else { artifacts.each { a -> @@ -375,7 +490,7 @@ allprojects { project -> version : dep.moduleVersion ?: '', classifier: a.classifier ?: '', ext : ext, - ], isProd, projPath) + ], isProd, projPath, outputSet) producedIds << aid // `a.file` downloads the artifact if not already cached, so scoping avoids fetching // artifacts the SBOM doesn't reference. Per-artifact try/catch: a single download can @@ -569,22 +684,111 @@ allprojects { project -> directIds.addAll(direct) } + // A lenient view of this config's artifacts, limited to the modules the walk fetched files for (and + // which verification kept), plus project outputs when `withProjects`. + def fetchedView = { cfg, withProjects -> + def fetched = paths.keySet().findAll { id -> nodes[id]?.project == null }.collect { id -> gavOf(nodes[id].coord) } as Set + cfg.incoming.artifactView { + lenient(true) + componentFilter { cid -> + cid instanceof org.gradle.api.artifacts.component.ModuleComponentIdentifier ? + fetched.contains("${cid.group}:${cid.module}:${cid.version}".toString()) : withProjects + } + } + } + + // Node ids in the order the fetched-artifact view lists them; null before Gradle 4.0, which has no + // lenient artifact view. + def orderFromView = { cfg -> + def view + try { + view = fetchedView(cfg, true) + } catch (MissingMethodException e) { + return null + } + def idsOfProject = [:] + nodes.each { id, n -> if (n.project != null) { idsOfProject.get(n.project, []) << id } } + def idsByFile = [:] + paths.each { id, files -> files.each { f -> idsByFile.get(f, []) << id } } + def order = [] + view.artifacts.each { a -> + def cid = a.id.componentIdentifier + if (cid instanceof org.gradle.api.artifacts.component.ProjectComponentIdentifier) { + def b = cid.metaClass.respondsTo(cid, 'getBuild') ? cid.build : null + def buildPath = b == null ? ':' : b.metaClass.respondsTo(b, 'getBuildPath') ? b.buildPath : b.name + // An included build's project paths can repeat this build's; its artifacts are files. + if (buildPath == ':') { + order.addAll(idsOfProject[cid.projectPath] ?: []) + return + } + } + order.addAll(idsByFile[a.file.absolutePath] ?: []) + } + order + } + + // Before Gradle 4.0. The legacy listing cannot leave out the artifacts a populateFilesFor scope + // excludes, so a scoped run keeps the walk's order. + def orderFromLegacy = { lenientCfg -> + if (populateGavs != null) { + return [] + } + def artifacts + try { + artifacts = lenientCfg.artifacts + } catch (MissingPropertyException e) { + artifacts = lenientCfg.getArtifacts({ true } as org.gradle.api.specs.Spec) + } + artifacts.collect { a -> + def mv = a.moduleVersion.id + def projPath = projectByGav["${mv.group ?: ''}:${mv.name}:${mv.version ?: ''}".toString()] + coordId([groupId: mv.group ?: '', artifactId: mv.name, version: mv.version ?: '', + classifier: a.classifier ?: '', ext: effExt(projPath, a.extension)]) + } + } + + // Puts this config's nodes in the order Gradle lists its artifacts: its classpath order, which no + // walk of the dependency graph reproduces. Listing them fetches their files, so only a --with-files + // run reads it, over the artifacts it fetched; a plain run keeps the walk's order. + def orderByArtifacts = { cfg, lenientCfg -> + if (!withFiles) { + return + } + def order + try { + order = orderFromView(cfg) + if (order == null) { + order = orderFromLegacy(lenientCfg) + } + } catch (Exception e) { + println "[socket-facts] could not order the classpath of ${configName}: ${rawDetail(e).readLines().find()}" + return + } + def listed = new LinkedHashSet(order.findAll { id -> nodes.containsKey(id) }) + if (listed.isEmpty()) { + return + } + // A node the listing leaves out (a pom-only module, an artifact this run did not fetch) keeps its + // place in the walk; the listed nodes fill the remaining places in classpath order. + def next = listed.iterator() + def ordered = [:] + nodes.each { id, node -> + def slot = listed.contains(id) ? next.next() : id + ordered[slot] = nodes[slot] + } + nodes.clear() + nodes.putAll(ordered) + } + // Drops the files of this config's artifacts that failed verification and reports them. def checkVerification = { cfg -> def external = paths.keySet().findAll { id -> nodes[id].project == null } if (external.isEmpty()) { return } - def fetched = external.collect { id -> gavOf(nodes[id].coord) } as Set def message = null try { - cfg.incoming.artifactView { - lenient(true) - componentFilter { id -> - id instanceof org.gradle.api.artifacts.component.ModuleComponentIdentifier && - fetched.contains("${id.group}:${id.module}:${id.version}".toString()) - } - }.artifacts.each { it.file } + fetchedView(cfg, false).artifacts.each { it.file } } catch (Exception e) { message = verificationMessage(e) if (message == null) { @@ -714,6 +918,7 @@ allprojects { project -> if (verifyArtifacts) { checkVerification(cfg) } + orderByArtifacts(cfg, lenient) walked = true } catch (Exception e) { // A config-LEVEL resolution failure (a resolutionStrategy/beforeResolve rule that throws, @@ -772,9 +977,14 @@ rootProject { rp -> rec(['project', pi.path, pi.group ?: '', pi.name, pi.version ?: '', pi.dir]) pi.buildFiles.each { f -> rec(['projectBuild', pi.path, f]) } } + if (!pi.excluded || withFilesProjects) { + pi.outputs.each { set, o -> rec(['outputSet', pi.path, set, o.includes.join(',')]) } + } if (withFilesProjects) { - pi.sources.each { s -> rec(['projectSrc', pi.path, s]) } - pi.targets.each { t -> rec(['projectTgt', pi.path, t]) } + pi.outputs.each { set, o -> + o.sources.each { s -> rec(['projectSrc', pi.path, s, set]) } + o.targets.each { t -> rec(['projectTgt', pi.path, t, set]) } + } } } @@ -785,11 +995,13 @@ rootProject { rp -> int rootIdx = 0 perSub.each { rootKey, tree -> def rootId = (rootIdx++).toString() - rec(['root', rootId, tree.projectPath, tree.config, tree.prod ? '1' : '0']) + def classpath = state.classpathRoles[tree.projectPath]?.get(tree.config) + rec(['root', rootId, tree.projectPath, tree.config, tree.prod ? '1' : '0', classpath?.set ?: '', classpath?.role ?: '']) tree.nodes.each { coordId, node -> def c = node.coord rec(['node', rootId, coordId, c.groupId ?: '', c.artifactId ?: '', c.version ?: '', c.ext ?: '', - c.classifier ?: '', tree.direct.contains(coordId) ? '1' : '0', node.project ?: '']) + c.classifier ?: '', tree.direct.contains(coordId) ? '1' : '0', node.project ?: '', + (node.outputSets ?: []).join(',')]) node.children.each { childId -> rec(['edge', rootId, coordId, childId]) } def fs = tree.paths[coordId] if (fs) { (fs as List).sort().each { p -> rec(['file', rootId, coordId, p]) } } diff --git a/src/commands/manifest/scripts/socket-facts.plugin.scala b/src/commands/manifest/scripts/socket-facts.plugin.scala index 6a736865d..3d1eb03d8 100644 --- a/src/commands/manifest/scripts/socket-facts.plugin.scala +++ b/src/commands/manifest/scripts/socket-facts.plugin.scala @@ -73,6 +73,8 @@ object SocketFactsPlugin extends AutoPlugin { // Real artifact ext per build module, so an ext-less inter-project dep gets its true coordinate. val moduleExts = buildModuleExts(allRefs, extracted) + val outputSets: Map[ProjectRef, Seq[(String, Configuration)]] = + allRefs.map(r => r -> sourceConfigs(r, extracted)).toMap // Every project: `update` also runs each upstream project's update, which would fail strictly. lazy val lenientState = extracted.append(allRefs.map(r => updateConfiguration.in(r) ~= (c => withMissingOk(c))), st) @@ -93,15 +95,15 @@ object SocketFactsPlugin extends AutoPlugin { runUpdateResilient(updateTaskName, ref, extracted, st, lenientState, matcher, failures, unscannable, resolver) .foreach { case (report, unresolved, ivyUnfetched, updateFailed) => foldReport(report, unresolved, ref, extracted, matcher, scannedConfigs, withFiles, populateScope, moduleExts, - resolver, ivyUnfetched, updateFailed, failures).foreach { + outputSets, resolver, ivyUnfetched, updateFailed, failures).foreach { case (rootKey, tree) => perSub(rootKey) = tree } } } } - val moduleDirs: Map[String, (Seq[String], Seq[String])] = - if (withFiles) buildModuleDirs(allRefs, extracted, st) else Map.empty + val moduleDirs: Map[String, Seq[(String, Seq[String], Seq[String])]] = + if (withFiles) buildModuleDirs(allRefs, outputSets, extracted, st) else Map.empty // Where sbt recorded the project's own settings as defined (the root build.sbt for a subproject // defined there); positions outside the build are sbt defaults, plugins or our injected base. @@ -141,11 +143,19 @@ object SocketFactsPlugin extends AutoPlugin { (buildFilesByRef.getOrElse(ref, Nil) ++ baseBuildSbt(baseDir, rootCanonPath).map(relOf)).distinct.sorted .foreach(f => rec("projectBuild", ref.project, f)) } + // A configuration's classpath holds the code of the configurations it extends (Test: Compile). + if (!isExcludedRef(ref) || withFiles) { + val sets = outputSets(ref) + sets.foreach { case (set, c) => + val held = Classpaths.allConfigs(c).map(_.name).toSet + rec("outputSet", ref.project, set, sets.collect { case (o, oc) if oc.name != c.name && held(oc.name) => o }.mkString(",")) + } + } if (withFiles) { - moduleDirs.get(mid.organization + ":" + mid.name + ":" + ver).foreach { - case (sources, targets) => - sources.foreach(s => rec("projectSrc", ref.project, s)) - targets.foreach(t => rec("projectTgt", ref.project, t)) + moduleDirs.getOrElse(mid.organization + ":" + mid.name + ":" + ver, Nil).foreach { + case (set, sources, targets) => + sources.foreach(s => rec("projectSrc", ref.project, s, set)) + targets.foreach(t => rec("projectTgt", ref.project, t, set)) } } } @@ -157,12 +167,12 @@ object SocketFactsPlugin extends AutoPlugin { case (_, tree) => val rootId = rootIdx.toString rootIdx += 1 - rec("root", rootId, tree.projectKey, tree.config, if (tree.prod) "1" else "0") + rec("root", rootId, tree.projectKey, tree.config, if (tree.prod) "1" else "0", tree.outputSet, tree.role) tree.nodes.foreach { case (coordId, node) => val c = node.coord rec("node", rootId, coordId, c.org, c.name, c.version, c.ext, c.classifier, if (node.direct) "1" else "0", - node.project) + node.project, node.outputSets.mkString(",")) node.children.foreach(ch => rec("edge", rootId, coordId, ch)) node.targets.foreach(p => rec("file", rootId, coordId, p)) if (node.unfetched) rec("unfetched", rootId, coordId) @@ -202,27 +212,36 @@ object SocketFactsPlugin extends AutoPlugin { }.toMap } - // Absolute source roots + compiled-output dirs per build module, keyed by GAV. --with-files only; - // absolute because reachability locates an internal module's code on THIS machine (no registry jar). - // ALL of the project's configurations are scanned (not just Compile/Test) so custom ones - // (IntegrationTest, ...) reach the analysis; source-file-level exclusion is coana's job. + // A project's code by output set: Compile's as `main`, and each configuration with a class directory + // of its own (Test, IntegrationTest, ...) under its name. + private def sourceConfigs(ref: ProjectRef, extracted: Extracted): Seq[(String, Configuration)] = { + val configs = extracted.getOpt(thisProject.in(ref)).map(_.configurations).getOrElse(Seq(Compile, Test)) + val mainDir = extracted.getOpt(classDirectory.in(ref).in(Compile)) + ("main" -> Compile) +: configs.filter { c => + c.name != Compile.name && extracted.getOpt(classDirectory.in(ref).in(c)).exists(d => !mainDir.exists(_ == d)) + }.map(c => c.name -> c) + } + + // Absolute source roots + compiled-output dirs per build module and output set, keyed by GAV. + // --with-files only; absolute because reachability locates an internal module's code on THIS machine + // (no registry jar). Source-file-level exclusion is coana's job. private def buildModuleDirs( allRefs: Seq[ProjectRef], + outputSets: Map[ProjectRef, Seq[(String, Configuration)]], extracted: Extracted, state: State - ): Map[String, (Seq[String], Seq[String])] = { + ): Map[String, Seq[(String, Seq[String], Seq[String])]] = { allRefs.map { ref => val mid = rootIdOf(extracted, ref) val ver = if (mid.revision == null) "" else mid.revision - val configs: Seq[Configuration] = - extracted.getOpt(thisProject.in(ref)).map(_.configurations).getOrElse(Seq(Compile, Test)) - val sources = (configs.flatMap(c => extracted.getOpt(sourceDirectories.in(ref).in(c)).getOrElse(Nil)) ++ - baseDirSources(ref, configs, extracted, state)) - .map(_.getAbsolutePath).distinct.sorted - val targets = configs - .flatMap(c => extracted.getOpt(classDirectory.in(ref).in(c))) - .map(_.getAbsolutePath).distinct.sorted - (mid.organization + ":" + mid.name + ":" + ver) -> ((sources, targets)) + val sets = outputSets(ref).map { case (set, c) => + val sources = (extracted.getOpt(sourceDirectories.in(ref).in(c)).getOrElse(Nil) ++ + baseDirSources(ref, Seq(c), extracted, state)) + .map(_.getAbsolutePath).distinct.sorted + val targets = extracted.getOpt(classDirectory.in(ref).in(c)).toSeq.map(_.getAbsolutePath) + (set, sources, targets) + } + (mid.organization + ":" + mid.name + ":" + ver) -> sets }.toMap } @@ -489,6 +508,7 @@ object SocketFactsPlugin extends AutoPlugin { withFiles: Boolean, populateScope: Option[Set[String]], moduleExts: Map[String, String], + outputSets: Map[ProjectRef, Seq[(String, Configuration)]], resolver: Resolver, ivyUnfetched: IvyUnfetched, // sbt's own update failed on artifacts alone, so one it could not fetch is a failure even @@ -554,6 +574,19 @@ object SocketFactsPlugin extends AutoPlugin { case Some(gavs) => gavs.contains(gavKey(m)) } + // The configuration sbt resolves a configuration's classpath for: the first of its internal one + // (compile-internal, ...), Compile and Default the report has (Defaults.findClasspathConfig). + val reported = report.configurations.map(confName).toSet + def classpathConf(c: Configuration): Option[String] = { + val internal = extracted.getOpt(internalConfigurationMap.in(ref)).map(_(c).name).getOrElse(c.name) + Seq(internal, Compile.name, "default").find(reported) + } + // Each output set's runtime classpath, and Compile's, which adds the compile-only (Provided) entries. + val classpathRoles: Map[String, (String, String)] = + outputSets(ref).flatMap { case (set, c) => + classpathConf(if (set == "main") Runtime else c).map(_ -> ((set, "runtime"))) + }.toMap ++ classpathConf(Compile).map(_ -> (("main", "compile"))) + report.configurations.foreach { cr => val cfg = confName(cr) if (matcher(cfg)) { @@ -566,13 +599,17 @@ object SocketFactsPlugin extends AutoPlugin { confsOf(ref).find(_.name == cfg).foreach { conf => // Keyed by project id, as projects need not have distinct coordinates. + // A dependency on a build project brings in the output sets of the configurations it reaches. def buildModuleNode(r: ProjectRef): String = { midToIds.getOrElseUpdate(moduleKey(rootIdOf(extracted, r)), mutable.LinkedHashSet.empty[String]) += r.project - nodes.getOrElseUpdate(r.project, { + val node = nodes.getOrElseUpdate(r.project, { val node = new Node(buildModuleCoord(r)) node.project = r.project node }) + val confs = reachedConfs.getOrElse(r.project, Set.empty[String]) + node.outputSets ++= outputSets.getOrElse(r, Nil) + .collect { case (set, c) if confs(c.name) => set } r.project } val (edges, reached) = interProjectEdges(conf) @@ -632,7 +669,10 @@ object SocketFactsPlugin extends AutoPlugin { } } - if (nodes.nonEmpty) perRoot(ref.project + "::" + cfg) = new RootTree(ref.project, cfg, prod, nodes) + if (nodes.nonEmpty) { + val (set, role) = classpathRoles.getOrElse(cfg, ("", "")) + perRoot(ref.project + "::" + cfg) = new RootTree(ref.project, cfg, prod, set, role, nodes) + } } } perRoot @@ -1000,6 +1040,8 @@ object SocketFactsPlugin extends AutoPlugin { var direct = false // Id of the build project this node is, when it is one. var project = "" + // That project's output sets this dependency brings in. + val outputSets = mutable.LinkedHashSet.empty[String] // External artifact's resolved jar(s); --with-files only. val targets = mutable.TreeSet.empty[String] // An expected artifact was not fetched; --with-files only. @@ -1013,6 +1055,9 @@ object SocketFactsPlugin extends AutoPlugin { val projectKey: String, val config: String, val prod: Boolean, + // The output set whose runtime or compile classpath this root is, or "". + val outputSet: String, + val role: String, val nodes: mutable.LinkedHashMap[String, Node] ) } diff --git a/src/commands/manifest/scripts/test/gradle-compat/order/build.gradle b/src/commands/manifest/scripts/test/gradle-compat/order/build.gradle new file mode 100644 index 000000000..24a54c14b --- /dev/null +++ b/src/commands/manifest/scripts/test/gradle-compat/order/build.gradle @@ -0,0 +1,24 @@ +// Ordering fixture for ../../socket-facts.init.gradle: `a` pulls `a1`, so a walk of the dependency +// graph meets a, a1, b while Gradle puts a's dependencies after its own siblings on the classpath. +// The `printClasspath` task writes the order Gradle itself builds the runtime classpath in. +apply plugin: 'java' +group = 'demo' +version = '1.0.0' + +repositories { + maven { url file('localrepo').toURI() } +} + +def modern = org.gradle.util.GradleVersion.current() >= org.gradle.util.GradleVersion.version('3.4') +def runtimeName = modern ? 'runtimeClasspath' : 'runtime' +dependencies { + add(modern ? 'implementation' : 'compile', 'demo.lib:a:1.0') + add(modern ? 'implementation' : 'compile', 'demo.lib:b:1.0') +} + +tasks.create("printClasspath") { + doLast { + file(project.property('classpathFile')).text = + runtimeName + '\n' + configurations.getByName(runtimeName).files.collect { it.name }.join('\n') + '\n' + } +} diff --git a/src/commands/manifest/scripts/test/gradle-compat/order/settings.gradle b/src/commands/manifest/scripts/test/gradle-compat/order/settings.gradle new file mode 100644 index 000000000..0b8b772d2 --- /dev/null +++ b/src/commands/manifest/scripts/test/gradle-compat/order/settings.gradle @@ -0,0 +1 @@ +rootProject.name = 'gradle-compat-order' diff --git a/src/commands/manifest/scripts/test/gradle-compat/smoke-test-order.sh b/src/commands/manifest/scripts/test/gradle-compat/smoke-test-order.sh new file mode 100755 index 000000000..817f26818 --- /dev/null +++ b/src/commands/manifest/scripts/test/gradle-compat/smoke-test-order.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Run socket-facts.init.gradle with --with-files on the ordering fixture and assert the runtime +# classpath root lists its nodes in the order Gradle itself builds that classpath in, which the +# fixture's own `printClasspath` task reports. A plain graph walk would give a different order. +# +# Usage: smoke-test-order.sh /path/to/gradle +set -euo pipefail +HERE="$(cd "$(dirname "$0")" && pwd)" +GRADLE="${1:?usage: smoke-test-order.sh }" +INIT="$HERE/../../socket-facts.init.gradle" +PROJECT="$HERE/order" +RECORDS="$PROJECT/records.tsv" +CLASSPATH="$PROJECT/classpath.txt" +# shellcheck source=SCRIPTDIR/../compat-cache.sh +. "$HERE/../compat-cache.sh" +GUH="$SOCKET_COMPAT_CACHE/gradle-home" + +bash "$HERE/../make-stub-repo.sh" "$PROJECT/localrepo" 'demo.lib:a:1.0+demo.lib:a1:1.0' 'demo.lib:a1:1.0' 'demo.lib:b:1.0' +rm -rf "$GUH" "$RECORDS" "$CLASSPATH" "$PROJECT/.gradle" "$PROJECT/build" + +echo "+ $("$GRADLE" --version 2>/dev/null | sed -n 's/^Gradle //p' | head -1) (order)" +( cd "$PROJECT" && "$GRADLE" --no-daemon --offline -g "$GUH" --init-script "$INIT" \ + -Psocket.withFiles=true -Psocket.recordsFile="$RECORDS" -PclasspathFile="$CLASSPATH" \ + socketFacts printClasspath -q ) + +python3 - "$RECORDS" "$CLASSPATH" <<'PY' +import sys +rows = [l.rstrip('\n').split('\t') for l in open(sys.argv[1]) if l.strip()] +config, *jars = [l.strip() for l in open(sys.argv[2]) if l.strip()] +root = next((r[1] for r in rows if r[0] == 'root' and r[3] == config), None) +names = [f"{r[4]}-{r[5]}.jar" for r in rows if r[0] == 'node' and r[1] == root] +if root is None: + print(f"FAIL: no {config} root"); sys.exit(1) +if names != jars: + print(f"FAIL: {config} nodes {names} != Gradle's classpath order {jars}"); sys.exit(1) +print(f"PASS (order): {config} nodes in Gradle's classpath order {jars}") +PY diff --git a/src/commands/manifest/scripts/test/maven-compat/assert-classpath.py b/src/commands/manifest/scripts/test/maven-compat/assert-classpath.py new file mode 100644 index 000000000..15d0a60fa --- /dev/null +++ b/src/commands/manifest/scripts/test/maven-compat/assert-classpath.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +# Assert the `classpath` records the Coana Maven core extension emitted for the smoke project's app +# modules: each classpath in the order Maven builds it (a pre-order walk of the resolved graph, filtered +# by scope), with the output sets each reactor module's artifacts bring in. app keeps demo.ext:libdep, +# which Maven leaves only under lib's test-jar node while the walk reaches lib's jar node first. +# Matches on groupId:artifactId, so versions can change freely. +import sys + +rows = [l.rstrip('\n').split('\t') for l in open(sys.argv[1]) if l.strip()] + +EXPECTED = { + 'demo:app:1.0': { + 'main': [ + ('demo:mid', 'main'), ('demo:lib', 'main'), ('demo.ext:tool', ''), ('demo.ext:shared', ''), + ('demo.ext:conflict-prod', ''), ('demo.ext:libdep', ''), + ], + 'test': [ + ('demo:mid', 'main'), ('demo:lib', 'main,test'), ('demo.ext:tool', ''), ('demo.ext:harness', ''), + ('demo.ext:harness-core', ''), ('demo.ext:conflict-test', ''), ('demo.ext:shared', ''), + ('demo.ext:conflict-prod', ''), ('demo.ext:libdep', ''), + ], + }, + 'demo:app2:1.0': { + 'main': [('demo.ext:libdep', ''), ('demo:mid', 'main'), ('demo:lib', 'main')], + 'test': [('demo:lib', 'test,main'), ('demo.ext:libdep', ''), ('demo:mid', 'main')], + }, +} + +errors = [] +for project, classpaths in EXPECTED.items(): + for name, expected in classpaths.items(): + got = [(':'.join(r[3].split(':')[:2]), r[4] if len(r) > 4 else '') + for r in rows if r[0] == 'classpath' and r[1] == project and r[2] == name] + if got != expected: + errors.append(f"{project} {name}: {got} != {expected}") + +if errors: + print("FAIL:") + for e in errors: print(" -", e) + sys.exit(1) +print(f"PASS (classpath): {', '.join(EXPECTED)} main and test classpaths in Maven's order") diff --git a/src/commands/manifest/scripts/test/maven-compat/project/app2/pom.xml b/src/commands/manifest/scripts/test/maven-compat/project/app2/pom.xml new file mode 100644 index 000000000..c953f5daa --- /dev/null +++ b/src/commands/manifest/scripts/test/maven-compat/project/app2/pom.xml @@ -0,0 +1,26 @@ + + + 4.0.0 + + demo + root + 1.0 + + app2 + + + + demo + lib + 1.0 + test-jar + test + + + demo + mid + 1.0 + + + diff --git a/src/commands/manifest/scripts/test/maven-compat/project/app2/src/main/java/demo/App2.java b/src/commands/manifest/scripts/test/maven-compat/project/app2/src/main/java/demo/App2.java new file mode 100644 index 000000000..0bd66ee50 --- /dev/null +++ b/src/commands/manifest/scripts/test/maven-compat/project/app2/src/main/java/demo/App2.java @@ -0,0 +1,4 @@ +package demo; + +public class App2 { +} diff --git a/src/commands/manifest/scripts/test/maven-compat/project/pom.xml b/src/commands/manifest/scripts/test/maven-compat/project/pom.xml index c9dc686a9..5b4eceb6e 100644 --- a/src/commands/manifest/scripts/test/maven-compat/project/pom.xml +++ b/src/commands/manifest/scripts/test/maven-compat/project/pom.xml @@ -9,6 +9,7 @@ lib mid app + app2 8 diff --git a/src/commands/manifest/scripts/test/maven-compat/smoke-test-workspaces.sh b/src/commands/manifest/scripts/test/maven-compat/smoke-test-workspaces.sh index 116e871f1..7d9f83cfd 100755 --- a/src/commands/manifest/scripts/test/maven-compat/smoke-test-workspaces.sh +++ b/src/commands/manifest/scripts/test/maven-compat/smoke-test-workspaces.sh @@ -45,10 +45,10 @@ if tool != 'maven': errors.append(f"meta tool {tool!r} != 'maven'") if build_root != sys.argv[2]: errors.append(f"buildRoot {build_root!r} != {sys.argv[2]!r}") -expected_build_files = {'pom.xml', 'lib/pom.xml', 'mid/pom.xml', 'app/pom.xml'} +expected_build_files = {'pom.xml', 'lib/pom.xml', 'mid/pom.xml', 'app/pom.xml', 'app2/pom.xml'} if build_files != expected_build_files: errors.append(f"projectBuild files {sorted(build_files)} != {sorted(expected_build_files)}") -expected = {'root', 'lib', 'mid', 'app'} +expected = {'root', 'lib', 'mid', 'app', 'app2'} if set(projects) != expected: errors.append(f"expected reactor modules {sorted(expected)}, got {sorted(projects)}") for artifact in expected & set(projects): diff --git a/src/commands/manifest/scripts/test/maven-compat/smoke-test.sh b/src/commands/manifest/scripts/test/maven-compat/smoke-test.sh index be8dac3f4..91f5ad01a 100755 --- a/src/commands/manifest/scripts/test/maven-compat/smoke-test.sh +++ b/src/commands/manifest/scripts/test/maven-compat/smoke-test.sh @@ -8,8 +8,8 @@ # inter-module edge lines up with its `project` record); # - resolves a scope conflict so the retained node stays reachable from its parent root; # - every component is reachable from a direct dependency of its own root (no orphaned components); -# - keeps the dependencies Maven leaves only under a sibling module's test-jar (app reaches -# demo.ext:libdep under lib's test-jar node, after lib's jar node through mid); +# - states each module's main and test classpaths in Maven's order, keeping the dependencies Maven +# leaves only under a sibling module's test-jar (assert-classpath.py); # - materializes resolved external jars under -Dsocket.withFiles; # - scopes that materialization to -Dsocket.populateFilesFor (a newline-delimited GAV file). # @@ -55,14 +55,7 @@ echo "+ $("$MVN" -v 2>/dev/null | head -1)" compile ) python3 "$HERE/assert-records.py" "$RECORDS" -python3 - "$RECORDS" <<'PY' -import sys -rows = [l.rstrip('\n').split('\t') for l in open(sys.argv[1]) if l.strip()] -app = {r[1] for r in rows if r[0] == 'root' and r[2] == 'demo:app:1.0'} -if not any(r[0] == 'node' and r[1] in app and r[2].startswith('demo.ext:libdep:') for r in rows): - print("FAIL: demo.ext:libdep missing from app"); sys.exit(1) -print("PASS: demo.ext:libdep in app through lib's test-jar") -PY +python3 "$HERE/assert-classpath.py" "$RECORDS" python3 "$HERE/../assert-reachability.py" "$RECORDS" # Second run: scope --with-files to a single GAV and assert ONLY that artifact is materialized. diff --git a/src/commands/manifest/scripts/test/run-compat.sh b/src/commands/manifest/scripts/test/run-compat.sh index eaa8d7512..ad9cabe96 100755 --- a/src/commands/manifest/scripts/test/run-compat.sh +++ b/src/commands/manifest/scripts/test/run-compat.sh @@ -66,6 +66,7 @@ run_gradle() { bash "$HERE/gradle-compat/smoke-test-workspaces.sh" "$dir/bin/gradle" bash "$HERE/gradle-compat/smoke-test-exclude-base.sh" "$dir/bin/gradle" bash "$HERE/gradle-compat/smoke-test-failures.sh" "$dir/bin/gradle" + bash "$HERE/gradle-compat/smoke-test-order.sh" "$dir/bin/gradle" done }