From 927304702c55348b25019d04caab1f4003ae6c8f Mon Sep 17 00:00:00 2001 From: Chris Selzo Date: Mon, 26 Oct 2026 09:54:22 -0700 Subject: [PATCH 1/3] Add fixture-backed compile test behind the fixtures build tag go test -tags fixtures compiles every release tarball in $FIXTURES with and without --exclude-compile-time-packages, using a fake package compiler. The trimmed output must ship exactly the packages listed by job.MF files, record the rest under compile_time_packages without their blobs, and carry the trimmed version suffix. In both outputs, every compiled package has exactly one blob matching its manifest digest. When nothing is omitted, the trimmed output must be byte-identical to the output without the flag. FIXTURES_OUT keeps the compiled tarballs for inspection. Co-Authored-By: Claude Opus 5.5 --- releasetarball/fixtures_test.go | 432 ++++++++++++++++++++++++++++++++ 1 file changed, 432 insertions(+) create mode 100644 releasetarball/fixtures_test.go diff --git a/releasetarball/fixtures_test.go b/releasetarball/fixtures_test.go new file mode 100644 index 000000000..bf3be3a88 --- /dev/null +++ b/releasetarball/fixtures_test.go @@ -0,0 +1,432 @@ +//go:build fixtures + +package releasetarball_test + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "crypto/sha1" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "os" + "path/filepath" + "slices" + "strings" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/cloudfoundry/bosh-agent/v2/releasetarball" + "github.com/cloudfoundry/bosh-agent/v2/releasetarball/internal/fakes" + "github.com/cloudfoundry/bosh-agent/v2/settings/directories" + "github.com/cloudfoundry/bosh-cli/v7/release/manifest" + "gopkg.in/yaml.v3" +) + +var _ = Describe("Compile fixtures", func() { + fixtureDir := os.Getenv("FIXTURES") + if fixtureDir == "" { + It("requires FIXTURES", func() { + Fail("FIXTURES required") + }) + return + } + + fixtures, err := filepath.Glob(filepath.Join(fixtureDir, "*.tgz")) + if err != nil { + It("reads FIXTURES", func() { Fail(err.Error()) }) + return + } + slices.Sort(fixtures) + if len(fixtures) == 0 { + It("requires fixture tarballs", func() { Fail("FIXTURES must contain at least one *.tgz file") }) + return + } + + for _, fixture := range fixtures { + fixture := fixture + It(filepath.Base(fixture), func() { + input := prepareFixture(GinkgoT(), fixture) + inputManifest := mustReleaseManifest(input) + shipped := jobPackageNames(GinkgoT(), input) + allPackages := packageNames(inputManifest.Packages) + kept := shipped + dropped := difference(allPackages, shipped) + + workDir := GinkgoT().TempDir() + plainDir := filepath.Join(workDir, "plain") + flaggedDir := filepath.Join(workDir, "flagged") + blobs := directories.NewProvider(filepath.Join(workDir, "blobs")) + Expect(os.MkdirAll(blobs.BlobsDir(), 0o755)).To(Succeed()) + Expect(os.MkdirAll(plainDir, 0o755)).To(Succeed()) + Expect(os.MkdirAll(flaggedDir, 0o755)).To(Succeed()) + pkgCompiler := new(fakes.Compiler) + pkgCompiler.CompileCalls(fakeCompilation(blobs)) + + plainPath, plain := compileFixture(pkgCompiler, input, blobs.BlobsDir(), plainDir, false) + flaggedPath, flagged := compileFixture(pkgCompiler, input, blobs.BlobsDir(), flaggedDir, true) + Expect(packageNamesCompiled(flagged.CompiledPkgs)).To(Equal(kept)) + Expect(packageNamesCompileTime(flagged.CompileTimePackages)).To(Equal(dropped)) + assertCompiledBlobs(GinkgoT(), flaggedPath, flagged.CompiledPkgs) + assertCompiledBlobs(GinkgoT(), plainPath, plain.CompiledPkgs) + assertNoDroppedBlobs(GinkgoT(), flaggedPath, dropped) + assertKeptPackages(inputManifest, flagged, kept) + + if len(dropped) == 0 { + plainBytes, err := os.ReadFile(plainPath) + Expect(err).NotTo(HaveOccurred()) + flaggedBytes, err := os.ReadFile(flaggedPath) + Expect(err).NotTo(HaveOccurred()) + Expect(sha256.Sum256(flaggedBytes)).To(Equal(sha256.Sum256(plainBytes))) + } else { + wantVersion := inputManifest.Version + "+trimmed" + if strings.Contains(inputManifest.Version, "+") { + wantVersion = inputManifest.Version + ".trimmed" + } + Expect(flagged.Version).To(Equal(wantVersion)) + } + + copyFixtureOutputs(fixture, flaggedPath, plainPath) + GinkgoWriter.Printf("fixture %s: flagged=%s plain=%s kept=%d dropped=%v\n", filepath.Base(fixture), flaggedPath, plainPath, len(kept), dropped) + }) + } +}) + +func prepareFixture(t GinkgoTInterface, fixture string) string { + GinkgoHelper() + dir := t.TempDir() + extractFixture(t, fixture, dir) + manifestPath := filepath.Join(dir, "release.MF") + raw, err := os.ReadFile(manifestPath) + Expect(err).NotTo(HaveOccurred()) + var release manifest.Manifest + Expect(yaml.Unmarshal(raw, &release)).To(Succeed()) + if len(release.CompiledPkgs) > 0 && len(release.Packages) == 0 { + var doc map[string]any + Expect(yaml.Unmarshal(raw, &doc)).To(Succeed()) + for _, pkg := range release.CompiledPkgs { + release.Packages = append(release.Packages, manifest.PackageRef{ + Name: pkg.Name, Version: pkg.Version, Fingerprint: pkg.Fingerprint, SHA1: pkg.SHA1, Dependencies: slices.Clone(pkg.Dependencies), + }) + oldBlob := filepath.Join(dir, "compiled_packages", pkg.Name+".tgz") + newDir := filepath.Join(dir, "packages") + Expect(os.MkdirAll(newDir, 0o755)).To(Succeed()) + Expect(os.Rename(oldBlob, filepath.Join(newDir, pkg.Name+".tgz"))).To(Succeed()) + } + delete(doc, "compiled_packages") + packages := make([]any, 0, len(release.Packages)) + for _, pkg := range release.Packages { + packages = append(packages, map[string]any{ + "name": pkg.Name, "version": pkg.Version, "fingerprint": pkg.Fingerprint, + "sha1": pkg.SHA1, "dependencies": pkg.Dependencies, + }) + } + doc["packages"] = packages + converted, err := yaml.Marshal(doc) + Expect(err).NotTo(HaveOccurred()) + Expect(os.WriteFile(manifestPath, converted, 0o644)).To(Succeed()) + } + return createTarballFromDir(t, dir) +} + +func extractFixture(t GinkgoTInterface, fixture, dir string) { + GinkgoHelper() + f, err := os.Open(fixture) + Expect(err).NotTo(HaveOccurred()) + defer closeAndIgnoreError(f) + gz, err := gzip.NewReader(f) + Expect(err).NotTo(HaveOccurred()) + defer closeAndIgnoreError(gz) + tr := tar.NewReader(gz) + for { + h, err := tr.Next() + if err == io.EOF { + break + } + Expect(err).NotTo(HaveOccurred()) + clean := filepath.Clean(filepath.FromSlash(h.Name)) + Expect(filepath.IsLocal(clean)).To(BeTrue(), "invalid tar path %q", h.Name) + dst := filepath.Join(dir, clean) + switch h.Typeflag { + case tar.TypeDir: + Expect(os.MkdirAll(dst, 0o755)).To(Succeed()) + case tar.TypeReg: + Expect(os.MkdirAll(filepath.Dir(dst), 0o755)).To(Succeed()) + out, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644) + Expect(err).NotTo(HaveOccurred()) + _, copyErr := io.Copy(out, tr) + Expect(out.Close()).To(Succeed()) + Expect(copyErr).NotTo(HaveOccurred()) + default: + Fail(fmt.Sprintf("fixture contains unsupported tar entry %q", h.Name)) + } + } +} + +func createTarballFromDir(t GinkgoTInterface, dir string) string { + GinkgoHelper() + var directories, files []string + Expect(filepath.WalkDir(dir, func(p string, entry os.DirEntry, err error) error { + if err != nil { + return err + } + if p == dir { + return nil + } + rel, err := filepath.Rel(dir, p) + if err != nil { + return err + } + if entry.IsDir() { + directories = append(directories, filepath.ToSlash(rel)) + } else { + files = append(files, filepath.ToSlash(rel)) + } + return nil + })).To(Succeed()) + slices.Sort(directories) + slices.Sort(files) + output := filepath.Join(t.TempDir(), "input.tgz") + f, err := os.Create(output) + Expect(err).NotTo(HaveOccurred()) + gw := gzip.NewWriter(f) + tw := tar.NewWriter(gw) + for _, name := range directories { + Expect(tw.WriteHeader(&tar.Header{Name: name + "/", Mode: 0o755, Typeflag: tar.TypeDir})).To(Succeed()) + } + for _, name := range files { + data, err := os.ReadFile(filepath.Join(dir, filepath.FromSlash(name))) + Expect(err).NotTo(HaveOccurred()) + Expect(tw.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: int64(len(data)), Typeflag: tar.TypeReg})).To(Succeed()) + _, err = tw.Write(data) + Expect(err).NotTo(HaveOccurred()) + } + Expect(tw.Close()).To(Succeed()) + Expect(gw.Close()).To(Succeed()) + Expect(f.Close()).To(Succeed()) + return output +} + +func assertCompiledBlobs(t GinkgoTInterface, tarball string, packages []manifest.CompiledPackageRef) { + GinkgoHelper() + f, err := os.Open(tarball) + Expect(err).NotTo(HaveOccurred()) + defer closeAndIgnoreError(f) + gz, err := gzip.NewReader(f) + Expect(err).NotTo(HaveOccurred()) + defer closeAndIgnoreError(gz) + tr := tar.NewReader(gz) + entries := map[string][][]byte{} + for { + h, err := tr.Next() + if err == io.EOF { + break + } + Expect(err).NotTo(HaveOccurred()) + contents, err := io.ReadAll(tr) + Expect(err).NotTo(HaveOccurred()) + name := strings.TrimPrefix(h.Name, "./") + entries[name] = append(entries[name], contents) + } + for _, pkg := range packages { + name := "compiled_packages/" + pkg.Name + ".tgz" + contents := entries[name] + Expect(contents).To(HaveLen(1), name) + if strings.HasPrefix(pkg.SHA1, "sha256:") { + actual := sha256.Sum256(contents[0]) + Expect("sha256:"+hex.EncodeToString(actual[:])).To(Equal(pkg.SHA1), name) + } else { + actual := sha1.Sum(contents[0]) + Expect(hex.EncodeToString(actual[:])).To(Equal(pkg.SHA1), name) + } + } +} + +func compileFixture(pkgCompiler *fakes.Compiler, input, blobsDir, outputDir string, exclude bool) (string, manifest.Manifest) { + GinkgoHelper() + output, err := releasetarball.Compile(pkgCompiler, input, blobsDir, outputDir, "banana-slug/1.23", releasetarball.CompileOptions{ExcludeCompileTimePackages: exclude}) + Expect(err).NotTo(HaveOccurred()) + return output, mustReleaseManifest(output) +} + +type jobManifest struct { + Packages []string `yaml:"packages"` +} + +func jobPackageNames(t GinkgoTInterface, input string) []string { + GinkgoHelper() + f, err := os.Open(input) + Expect(err).NotTo(HaveOccurred()) + defer closeAndIgnoreError(f) + gz, err := gzip.NewReader(f) + Expect(err).NotTo(HaveOccurred()) + defer closeAndIgnoreError(gz) + tr := tar.NewReader(gz) + var names []string + for { + h, err := tr.Next() + if err == io.EOF { + break + } + Expect(err).NotTo(HaveOccurred()) + if !strings.HasPrefix(h.Name, "jobs/") || !strings.HasSuffix(h.Name, ".tgz") { + continue + } + jobBytes, err := io.ReadAll(tr) + Expect(err).NotTo(HaveOccurred()) + jobGz, err := gzip.NewReader(bytes.NewReader(jobBytes)) + Expect(err).NotTo(HaveOccurred()) + jobTar := tar.NewReader(jobGz) + for { + jobHeader, err := jobTar.Next() + if err == io.EOF { + break + } + Expect(err).NotTo(HaveOccurred()) + if strings.TrimPrefix(jobHeader.Name, "./") != "job.MF" { + continue + } + var job jobManifest + Expect(yaml.NewDecoder(jobTar).Decode(&job)).To(Succeed()) + names = append(names, job.Packages...) + } + Expect(jobGz.Close()).To(Succeed()) + } + return uniqueSorted(names) +} + +func assertKeptPackages(source, output manifest.Manifest, kept []string) { + GinkgoHelper() + sourcePackages := map[string]manifest.PackageRef{} + for _, pkg := range source.Packages { + sourcePackages[pkg.Name] = pkg + } + keptSet := make(map[string]bool, len(kept)) + for _, name := range kept { + keptSet[name] = true + } + for _, compiled := range output.CompiledPkgs { + original := sourcePackages[compiled.Name] + var dependencies []string + for _, dependency := range original.Dependencies { + if keptSet[dependency] { + dependencies = append(dependencies, dependency) + } + } + Expect(slices.Equal(compiled.Dependencies, dependencies)).To(BeTrue(), compiled.Name) + if dependencyClosureContainsDropped(original.Name, sourcePackages, keptSet, map[string]bool{}) { + Expect(compiled.OriginalFingerprint).NotTo(BeEmpty(), compiled.Name) + Expect(compiled.OriginalFingerprint).NotTo(Equal(compiled.Fingerprint), compiled.Name) + Expect(compiled.Version).To(Equal(compiled.Fingerprint), compiled.Name) + } else { + Expect(compiled.OriginalFingerprint).To(BeEmpty(), compiled.Name) + Expect(compiled.Version).To(Equal(original.Version), compiled.Name) + Expect(compiled.Fingerprint).To(Equal(original.Fingerprint), compiled.Name) + } + } +} + +func dependencyClosureContainsDropped(name string, packages map[string]manifest.PackageRef, kept, visited map[string]bool) bool { + if visited[name] { + return false + } + visited[name] = true + for _, dependency := range packages[name].Dependencies { + if !kept[dependency] || dependencyClosureContainsDropped(dependency, packages, kept, visited) { + return true + } + } + return false +} + +func assertNoDroppedBlobs(t GinkgoTInterface, tarball string, dropped []string) { + GinkgoHelper() + f, err := os.Open(tarball) + Expect(err).NotTo(HaveOccurred()) + defer closeAndIgnoreError(f) + gz, err := gzip.NewReader(f) + Expect(err).NotTo(HaveOccurred()) + defer closeAndIgnoreError(gz) + tr := tar.NewReader(gz) + entries := map[string]bool{} + for { + h, err := tr.Next() + if err == io.EOF { + break + } + Expect(err).NotTo(HaveOccurred()) + entries[h.Name] = true + } + for _, name := range dropped { + Expect(entries["compiled_packages/"+name+".tgz"]).To(BeFalse(), name) + } +} + +func copyFixtureOutputs(fixture, flagged, plain string) { + GinkgoHelper() + outputDir := os.Getenv("FIXTURES_OUT") + if outputDir == "" { + return + } + Expect(os.MkdirAll(outputDir, 0o755)).To(Succeed()) + base := filepath.Base(fixture) + for _, output := range []struct{ source, suffix string }{{flagged, ".flagged.tgz"}, {plain, ".plain.tgz"}} { + data, err := os.ReadFile(output.source) + Expect(err).NotTo(HaveOccurred()) + Expect(os.WriteFile(filepath.Join(outputDir, strings.TrimSuffix(base, ".tgz")+output.suffix), data, 0o644)).To(Succeed()) + } +} + +func packageNames(packages []manifest.PackageRef) []string { + names := make([]string, 0, len(packages)) + for _, pkg := range packages { + names = append(names, pkg.Name) + } + return uniqueSorted(names) +} + +func packageNamesCompiled(packages []manifest.CompiledPackageRef) []string { + names := make([]string, 0, len(packages)) + for _, pkg := range packages { + names = append(names, pkg.Name) + } + return uniqueSorted(names) +} + +func packageNamesCompileTime(packages []manifest.CompileTimePackageRef) []string { + names := make([]string, 0, len(packages)) + for _, pkg := range packages { + names = append(names, pkg.Name) + } + return uniqueSorted(names) +} + +func uniqueSorted(names []string) []string { + set := map[string]bool{} + for _, name := range names { + set[name] = true + } + result := make([]string, 0, len(set)) + for name := range set { + result = append(result, name) + } + slices.Sort(result) + return result +} + +func difference(left, right []string) []string { + rightSet := map[string]bool{} + for _, name := range right { + rightSet[name] = true + } + var result []string + for _, name := range left { + if !rightSet[name] { + result = append(result, name) + } + } + return uniqueSorted(result) +} From 0efc4b34786564966e335d670bec1ff306c610d6 Mon Sep 17 00:00:00 2001 From: Nishad Mathur Date: Wed, 28 Oct 2026 18:12:25 -0700 Subject: [PATCH 2/3] Simplify compile-time package trimming and its tests Build dependency keys and detect trimmed dependencies in one traversal. Remove redundant writer state and validate job packages as their manifests are read. Use Ginkgo tables, explicit fixtures and gstruct assertions; consolidate archive test helpers. Preserve existing identity vectors and keep the complete non-vendored production diff within 300 changed lines. --- main/compile.go | 8 +- releasetarball/archive_helpers_test.go | 286 ++++++++++++++ releasetarball/compile.go | 18 +- releasetarball/compile_test.go | 368 +----------------- releasetarball/export_test.go | 30 +- releasetarball/fixtures_test.go | 143 +++---- releasetarball/identity.go | 134 ++----- releasetarball/identity_test.go | 46 ++- releasetarball/shipped.go | 47 +-- .../onsi/gomega/gstruct/elements.go | 235 +++++++++++ .../gomega/gstruct/errors/nested_types.go | 72 ++++ .../github.com/onsi/gomega/gstruct/fields.go | 187 +++++++++ .../github.com/onsi/gomega/gstruct/ignore.go | 41 ++ vendor/github.com/onsi/gomega/gstruct/keys.go | 126 ++++++ .../github.com/onsi/gomega/gstruct/pointer.go | 59 +++ .../github.com/onsi/gomega/gstruct/types.go | 19 + vendor/modules.txt | 2 + 17 files changed, 1227 insertions(+), 594 deletions(-) create mode 100644 releasetarball/archive_helpers_test.go create mode 100644 vendor/github.com/onsi/gomega/gstruct/elements.go create mode 100644 vendor/github.com/onsi/gomega/gstruct/errors/nested_types.go create mode 100644 vendor/github.com/onsi/gomega/gstruct/fields.go create mode 100644 vendor/github.com/onsi/gomega/gstruct/ignore.go create mode 100644 vendor/github.com/onsi/gomega/gstruct/keys.go create mode 100644 vendor/github.com/onsi/gomega/gstruct/pointer.go create mode 100644 vendor/github.com/onsi/gomega/gstruct/types.go diff --git a/main/compile.go b/main/compile.go index 0a6bbd6c8..7e825d525 100644 --- a/main/compile.go +++ b/main/compile.go @@ -41,7 +41,7 @@ func compileTarball(command string, args []string) { log.Fatal(err) } for _, releaseTarballPath := range options.SourceReleases { - compiledReleaseTarballPath, err := releasetarball.Compile(compiler, releaseTarballPath, dirProvider.BlobsDir(), options.OutputDirectory, compiledReleaseFileSuffix, releasetarball.CompileOptions{ExcludeCompileTimePackages: options.ExcludeCompileTimePackages}) + compiledReleaseTarballPath, err := releasetarball.Compile(compiler, releaseTarballPath, dirProvider.BlobsDir(), options.OutputDirectory, compiledReleaseFileSuffix, options.CompileOptions) if err != nil { log.Fatal(err) } @@ -50,9 +50,9 @@ func compileTarball(command string, args []string) { } type CompileTarballOptions struct { - OutputDirectory string - SourceReleases []string - ExcludeCompileTimePackages bool + OutputDirectory string + SourceReleases []string + releasetarball.CompileOptions } func newCompileTarballOptions(command string, args []string) (CompileTarballOptions, error) { diff --git a/releasetarball/archive_helpers_test.go b/releasetarball/archive_helpers_test.go new file mode 100644 index 000000000..63fc548e4 --- /dev/null +++ b/releasetarball/archive_helpers_test.go @@ -0,0 +1,286 @@ +package releasetarball_test + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "crypto/sha1" + "encoding/hex" + "errors" + "fmt" + "io" + "log" + "maps" + "os" + "path/filepath" + "strings" + "time" + + "github.com/cloudfoundry/bosh-agent/v2/agent/applier/models" + "github.com/cloudfoundry/bosh-agent/v2/agent/compiler" + "github.com/cloudfoundry/bosh-agent/v2/releasetarball" + "github.com/cloudfoundry/bosh-agent/v2/settings/directories" + "github.com/cloudfoundry/bosh-cli/v7/release/manifest" + boshcrypto "github.com/cloudfoundry/bosh-utils/crypto" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gopkg.in/yaml.v3" +) + +func closeAndIgnoreError(c io.Closer) { + _ = c.Close() //nolint:errcheck +} + +type writeTarballFileFunc func(tw *tar.Writer) error + +func createTGZ(functions ...writeTarballFileFunc) ([]byte, error) { + fn := func(w io.Writer) error { + gw := gzip.NewWriter(w) + defer closeAndIgnoreError(gw) + tw := tar.NewWriter(gw) + defer closeAndIgnoreError(tw) + for _, fn := range functions { + if err := fn(tw); err != nil { + return err + } + } + return nil + } + w := new(bytes.Buffer) + err := fn(w) + return w.Bytes(), err +} + +func simpleFile(name string, content []byte, mode int64) writeTarballFileFunc { + return func(tw *tar.Writer) error { + if err := tw.WriteHeader(newSimpleFileHeader(name, int64(len(content)), mode)); err != nil { + return err + } + _, err := tw.Write(content) + if err != nil { + return err + } + return nil + } +} + +func newSimpleFileHeader(name string, length, mode int64) *tar.Header { + t := time.Unix(0, 0).In(time.UTC) + return &tar.Header{ + Typeflag: tar.TypeReg, + Name: name, + Size: length, + ModTime: t, + AccessTime: t, + ChangeTime: t, + Mode: mode, + } +} + +func listFileNamesInTarball(t GinkgoTInterface, filePath string) []tar.Header { + f, err := os.Open(filePath) + if err != nil { + t.Fatal(err) + } + defer closeAndIgnoreError(f) + gr, err := gzip.NewReader(f) + if err != nil { + log.Fatal(err) + } + tr := tar.NewReader(gr) + var infos []tar.Header + for { + h, err := tr.Next() + if err != nil { + if errors.Is(err, io.EOF) { + break + } + t.Fatal(err) + } + cp := *h + cp.PAXRecords = maps.Clone(h.PAXRecords) + infos = append(infos, cp) + _, _ = io.Copy(io.Discard, tr) //nolint:errcheck + } + return infos +} + +func fakeCompilation(d directories.Provider) func(c compiler.Package, packages []models.Package) (string, boshcrypto.Digest, error) { + return func(c compiler.Package, packages []models.Package) (string, boshcrypto.Digest, error) { + blobContent, err := createTGZ(simpleFile("packaging", fmt.Appendf(nil, `"echo Compiled %q`, c.Name), 0o0744)) + if err != nil { + log.Fatal(err) + } + compiledBlobstoreID := fmt.Sprintf("%s-compiled-blob", c.Name) + if err := os.WriteFile(filepath.Join(d.BlobsDir(), compiledBlobstoreID), blobContent, 0o0644); err != nil { + log.Fatal(err) + } + digester := sha1.New() + _, _ = digester.Write(blobContent) + digest := boshcrypto.NewDigest(boshcrypto.DigestAlgorithmSHA1, hex.EncodeToString(digester.Sum(nil))) + return compiledBlobstoreID, digest, nil + } +} + +func writePlainTarJobRelease(t GinkgoTInterface, release manifest.Manifest, jobMF string) string { + return writeRawJobRelease(t, release, "job", createTar(simpleFile("job.MF", []byte(jobMF), 0o644))) +} + +func writeRawJobRelease(t GinkgoTInterface, release manifest.Manifest, jobName string, inner []byte) string { + GinkgoHelper() + releaseMF, err := yaml.Marshal(release) + Expect(err).NotTo(HaveOccurred()) + archive, err := createTGZ(simpleFile("release.MF", releaseMF, 0o644), simpleFile("jobs/"+jobName+".tgz", inner, 0o644)) + Expect(err).NotTo(HaveOccurred()) + path := filepath.Join(t.TempDir(), "release.tgz") + Expect(os.WriteFile(path, archive, 0o644)).To(Succeed()) + return path +} + +func createTar(functions ...writeTarballFileFunc) []byte { + var buffer bytes.Buffer + tw := tar.NewWriter(&buffer) + for _, fn := range functions { + Expect(fn(tw)).To(Succeed()) + } + Expect(tw.Close()).To(Succeed()) + return buffer.Bytes() +} + +func writeShippedRelease(t GinkgoTInterface, release manifest.Manifest, jobs map[string]string, dotPrefix bool) string { + releaseMF, err := yaml.Marshal(release) + Expect(err).NotTo(HaveOccurred()) + files := []writeTarballFileFunc{simpleFile("release.MF", releaseMF, 0o644)} + for name, jobMF := range jobs { + innerName := "job.MF" + outerName := "jobs/" + name + ".tgz" + if dotPrefix { + innerName = "./" + innerName + outerName = "./" + outerName + } + var inner []byte + switch jobMF { + case "invalid-archive": + inner = []byte("invalid nested tar") + case "missing": + inner, err = createTGZ() + default: + inner, err = createTGZ(simpleFile(innerName, []byte(jobMF), 0o644)) + } + Expect(err).NotTo(HaveOccurred()) + files = append(files, simpleFile(outerName, inner, 0o644)) + } + archive, err := createTGZ(files...) + Expect(err).NotTo(HaveOccurred()) + path := filepath.Join(t.TempDir(), "release.tgz") + Expect(os.WriteFile(path, archive, 0o644)).To(Succeed()) + return path +} + +func mustReleaseManifest(path string) manifest.Manifest { + result, err := releasetarball.Manifest(path) + Expect(err).NotTo(HaveOccurred()) + return result +} + +// rewriteReleaseArchive keeps archive setup in one place for malformed-input tests. +func rewriteReleaseArchive(t GinkgoTInterface, source string, rewrite func(string, []byte) ([]byte, bool)) string { + GinkgoHelper() + f, err := os.Open(source) + Expect(err).NotTo(HaveOccurred()) + defer closeAndIgnoreError(f) + gr, err := gzip.NewReader(f) + Expect(err).NotTo(HaveOccurred()) + defer closeAndIgnoreError(gr) + tr := tar.NewReader(gr) + var files []writeTarballFileFunc + for { + h, err := tr.Next() + if errors.Is(err, io.EOF) { + break + } + Expect(err).NotTo(HaveOccurred()) + if h.FileInfo().IsDir() { + continue + } + contents, err := io.ReadAll(tr) + Expect(err).NotTo(HaveOccurred()) + if contents, keep := rewrite(h.Name, contents); keep { + files = append(files, simpleFile(h.Name, contents, h.Mode)) + } + } + archive, err := createTGZ(files...) + Expect(err).NotTo(HaveOccurred()) + path := filepath.Join(t.TempDir(), "release.tgz") + Expect(os.WriteFile(path, archive, 0o644)).To(Succeed()) + return path +} + +func rewriteReleaseManifest(t GinkgoTInterface, source string, replacement manifest.Manifest) string { + return rewriteReleaseArchive(t, source, func(name string, contents []byte) ([]byte, bool) { + if filepath.Base(name) == "release.MF" { + var err error + contents, err = yaml.Marshal(replacement) + Expect(err).NotTo(HaveOccurred()) + } + return contents, true + }) +} + +func rewriteJobPackages(t GinkgoTInterface, source string, packageNames []string) string { + return rewriteReleaseArchive(t, source, func(name string, contents []byte) ([]byte, bool) { + if filepath.Base(filepath.Dir(name)) == "jobs" && filepath.Ext(name) == ".tgz" { + jobName := strings.TrimSuffix(filepath.Base(name), ".tgz") + jobMF := fmt.Sprintf("name: %s\npackages: [%s]\n", jobName, strings.Join(packageNames, ", ")) + var err error + contents, err = createTGZ(simpleFile("job.MF", []byte(jobMF), 0o644)) + Expect(err).NotTo(HaveOccurred()) + } + return contents, true + }) +} + +func removeJobArchives(t GinkgoTInterface, source string) string { + return rewriteJobArchives(t, source, true) +} + +func replaceJobArchives(t GinkgoTInterface, source string) string { + return rewriteJobArchives(t, source, false) +} + +func rewriteJobArchives(t GinkgoTInterface, source string, remove bool) string { + return rewriteReleaseArchive(t, source, func(name string, contents []byte) ([]byte, bool) { + if filepath.Base(filepath.Dir(name)) == "jobs" && filepath.Ext(name) == ".tgz" { + if remove { + return nil, false + } + var err error + contents, err = createTGZ() + Expect(err).NotTo(HaveOccurred()) + } + return contents, true + }) +} + +func releaseManifestBytes(filePath string) ([]byte, error) { + f, err := os.Open(filePath) + if err != nil { + return nil, err + } + defer closeAndIgnoreError(f) + gr, err := gzip.NewReader(f) + if err != nil { + return nil, err + } + defer closeAndIgnoreError(gr) + tr := tar.NewReader(gr) + for { + h, err := tr.Next() + if err != nil { + return nil, err + } + if filepath.Base(h.Name) == "release.MF" { + return io.ReadAll(tr) + } + } +} diff --git a/releasetarball/compile.go b/releasetarball/compile.go index d1c857110..5d888cf4a 100644 --- a/releasetarball/compile.go +++ b/releasetarball/compile.go @@ -117,7 +117,7 @@ func Compile(compiler boshcomp.Compiler, boshReleaseTarballPath, blobsDirectory, log.Printf("Finished packages compilation after %s", time.Since(start)) log.Printf("Archiving compiled BOSH Release %s/%s with stemcell %s", m.Name, m.Version, stemcellSlug) - return writeCompiledRelease(m, outputDirectory, stemcellSlug, blobsDirectory, boshReleaseTarballPath, m.Packages, compiledPackages, options, shipped) + return writeCompiledRelease(m, outputDirectory, stemcellSlug, blobsDirectory, boshReleaseTarballPath, compiledPackages, shipped) } func compilePackage(compiledPackages []boshmodels.Package, p manifest.PackageRef, blobstoreIDs map[string]string, compiler boshcomp.Compiler) ([]boshmodels.Package, error) { @@ -209,16 +209,16 @@ func newCompiledPackageRef(pkg boshmodels.Package, src manifest.PackageRef, stem return compiled } -func writeCompiledRelease(m manifest.Manifest, outputDirectory, stemcellFilenameSuffix, blobsDirectory, initialTarball string, sourcePackages []manifest.PackageRef, compiledPackages []boshmodels.Package, options CompileOptions, shipped map[string]bool) (string, error) { - identities := map[string]rewrittenPkg{} - if options.ExcludeCompileTimePackages { +func writeCompiledRelease(m manifest.Manifest, outputDirectory, stemcellFilenameSuffix, blobsDirectory, initialTarball string, compiledPackages []boshmodels.Package, shipped map[string]bool) (string, error) { + var identities map[string]rewrittenPkg + if shipped != nil { m.CompileTimePackages = nil var err error - identities, err = rewriteIdentities(sourcePackages, shipped) + identities, err = rewriteIdentities(m.Packages, shipped) if err != nil { return "", err } - for _, p := range sourcePackages { + for _, p := range m.Packages { if shipped[p.Name] { continue } @@ -239,14 +239,14 @@ func writeCompiledRelease(m manifest.Manifest, outputDirectory, stemcellFilename keptCompiledPackages := make([]boshmodels.Package, 0, len(compiledPackages)) m.CompiledPkgs = make([]manifest.CompiledPackageRef, 0, len(compiledPackages)) for _, p := range compiledPackages { - if options.ExcludeCompileTimePackages && !shipped[p.Name] { + if shipped != nil && !shipped[p.Name] { continue } keptCompiledPackages = append(keptCompiledPackages, p) - srcIndex := slices.IndexFunc(sourcePackages, func(ref manifest.PackageRef) bool { + srcIndex := slices.IndexFunc(m.Packages, func(ref manifest.PackageRef) bool { return ref.Name == p.Name && ref.Version == p.Version }) - compiledRef := newCompiledPackageRef(p, sourcePackages[srcIndex], stemcellFilenameSuffix) + compiledRef := newCompiledPackageRef(p, m.Packages[srcIndex], stemcellFilenameSuffix) if identity := identities[p.Name]; identity.Rewritten { compiledRef.Version = identity.Fingerprint compiledRef.Fingerprint = identity.Fingerprint diff --git a/releasetarball/compile_test.go b/releasetarball/compile_test.go index a67a8d0cc..6595c7ee3 100644 --- a/releasetarball/compile_test.go +++ b/releasetarball/compile_test.go @@ -4,28 +4,24 @@ import ( "archive/tar" "bytes" "compress/gzip" - "crypto/sha1" - "encoding/hex" "errors" "fmt" "io" "log" - "maps" "os" "path/filepath" "strings" "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + . "github.com/onsi/gomega/gstruct" "github.com/cloudfoundry/bosh-cli/v7/release/manifest" boshcrypto "github.com/cloudfoundry/bosh-utils/crypto" "gopkg.in/yaml.v3" "github.com/cloudfoundry/bosh-agent/v2/agent/applier/models" - "github.com/cloudfoundry/bosh-agent/v2/agent/compiler" "github.com/cloudfoundry/bosh-agent/v2/releasetarball" "github.com/cloudfoundry/bosh-agent/v2/releasetarball/internal/fakes" "github.com/cloudfoundry/bosh-agent/v2/settings/directories" @@ -136,11 +132,12 @@ var _ = Describe("Compile", func() { HaveField("Name", "log-cache"), HaveField("Name", "log-cache-cf-auth-proxy"), HaveField("Name", "log-cache-gateway"), HaveField("Name", "log-cache-syslog-server"), )) - Expect(compiled.CompileTimePackages).To(HaveLen(1)) - Expect(compiled.CompileTimePackages[0].Name).To(Equal(source.Packages[0].Name)) - Expect(compiled.CompileTimePackages[0].Version).To(Equal(source.Packages[0].Version)) - Expect(compiled.CompileTimePackages[0].Fingerprint).To(Equal(source.Packages[0].Fingerprint)) - Expect(compiled.CompileTimePackages[0].Dependencies).To(Equal(source.Packages[0].Dependencies)) + Expect(compiled.CompileTimePackages).To(ConsistOf(MatchAllFields(Fields{ + "Name": Equal(source.Packages[0].Name), + "Version": Equal(source.Packages[0].Version), + "Fingerprint": Equal(source.Packages[0].Fingerprint), + "Dependencies": Equal(source.Packages[0].Dependencies), + }))) rawManifest, err := releaseManifestBytes(resultPath) Expect(err).NotTo(HaveOccurred()) if capturePath := os.Getenv("CAPTURE_COMPILE_MANIFEST"); capturePath != "" { @@ -301,42 +298,19 @@ var _ = Describe("Compile", func() { Expect(entries).To(BeEmpty()) }) - It("fails before compilation or output for missing job archives, missing job.MF, and an empty shipped set", func() { - fixture := filepath.Join("testdata", "log-cache-release-3.0.9.tgz") - tests := []struct { - name string - input func() string - want string - }{ - { - name: "missing archive", - input: func() string { return removeJobArchives(GinkgoT(), fixture) }, - want: "missing job archive", - }, - { - name: "missing job.MF", - input: func() string { return replaceJobArchives(GinkgoT(), fixture) }, - want: "no job.MF", - }, - { - name: "empty shipped set", - input: func() string { return rewriteJobPackages(GinkgoT(), fixture, nil) }, - want: "no shipped packages", - }, - } - for _, tt := range tests { - By(tt.name) - output := GinkgoT().TempDir() - pkgCompiler = new(fakes.Compiler) - pkgCompiler.CompileCalls(fakeCompilation(d)) - _, err := releasetarball.Compile(pkgCompiler, tt.input(), d.BlobsDir(), output, stemcellSlug, releasetarball.CompileOptions{ExcludeCompileTimePackages: true}) - Expect(err).To(MatchError(ContainSubstring(tt.want))) - Expect(pkgCompiler.CompileCallCount()).To(BeZero()) - entries, err := os.ReadDir(output) - Expect(err).NotTo(HaveOccurred()) - Expect(entries).To(BeEmpty()) - } - }) + DescribeTable("fails before compilation or output for invalid job archives", func(rewrite func(GinkgoTInterface, string) string, want string) { + input := rewrite(GinkgoT(), filepath.Join("testdata", "log-cache-release-3.0.9.tgz")) + _, err := releasetarball.Compile(pkgCompiler, input, d.BlobsDir(), releasesOutputDir, stemcellSlug, releasetarball.CompileOptions{ExcludeCompileTimePackages: true}) + Expect(err).To(MatchError(ContainSubstring(want))) + Expect(pkgCompiler.CompileCallCount()).To(BeZero()) + Expect(os.ReadDir(releasesOutputDir)).To(BeEmpty()) + }, + Entry("missing archive", removeJobArchives, "missing job archive"), + Entry("missing job.MF", replaceJobArchives, "no job.MF"), + Entry("empty shipped set", func(t GinkgoTInterface, input string) string { + return rewriteJobPackages(t, input, nil) + }, "no shipped packages"), + ) When("compiling a tarball with a package dependency cycle", func() { BeforeEach(func() { @@ -523,101 +497,6 @@ func assertPackageFields(compiledManifest, sourceManifest manifest.Manifest, ind Expect(compiledManifest.CompiledPkgs[index].Version).To(Equal(compiledManifest.CompiledPkgs[index].Fingerprint), "this seems to be equal on releases exported from a director") } -func closeAndIgnoreError(c io.Closer) { - _ = c.Close() //nolint:errcheck -} - -type writeTarballFileFunc func(tw *tar.Writer) error - -func createTGZ(functions ...writeTarballFileFunc) ([]byte, error) { - fn := func(w io.Writer) error { - gw := gzip.NewWriter(w) - defer closeAndIgnoreError(gw) - tw := tar.NewWriter(gw) - defer closeAndIgnoreError(tw) - for _, fn := range functions { - if err := fn(tw); err != nil { - return err - } - } - return nil - } - w := new(bytes.Buffer) - err := fn(w) - return w.Bytes(), err -} - -func simpleFile(name string, content []byte, mode int64) writeTarballFileFunc { - return func(tw *tar.Writer) error { - if err := tw.WriteHeader(newSimpleFileHeader(name, int64(len(content)), mode)); err != nil { - return err - } - _, err := tw.Write(content) - if err != nil { - return err - } - return nil - } -} - -func newSimpleFileHeader(name string, length, mode int64) *tar.Header { - t := time.Unix(0, 0).In(time.UTC) - return &tar.Header{ - Typeflag: tar.TypeReg, - Name: name, - Size: length, - ModTime: t, - AccessTime: t, - ChangeTime: t, - Mode: mode, - } -} - -func listFileNamesInTarball(t GinkgoTInterface, filePath string) []tar.Header { - f, err := os.Open(filePath) - if err != nil { - t.Fatal(err) - } - defer closeAndIgnoreError(f) - gr, err := gzip.NewReader(f) - if err != nil { - log.Fatal(err) - } - tr := tar.NewReader(gr) - var infos []tar.Header - for { - h, err := tr.Next() - if err != nil { - if errors.Is(err, io.EOF) { - break - } - t.Fatal(err) - } - cp := *h - cp.PAXRecords = maps.Clone(h.PAXRecords) - infos = append(infos, cp) - _, _ = io.Copy(io.Discard, tr) //nolint:errcheck - } - return infos -} - -func fakeCompilation(d directories.Provider) func(c compiler.Package, packages []models.Package) (string, boshcrypto.Digest, error) { - return func(c compiler.Package, packages []models.Package) (string, boshcrypto.Digest, error) { - blobContent, err := createTGZ(simpleFile("packaging", fmt.Appendf(nil, `"echo Compiled %q`, c.Name), 0o0744)) - if err != nil { - log.Fatal(err) - } - compiledBlobstoreID := fmt.Sprintf("%s-compiled-blob", c.Name) - if err := os.WriteFile(filepath.Join(d.BlobsDir(), compiledBlobstoreID), blobContent, 0o0644); err != nil { - log.Fatal(err) - } - digester := sha1.New() - _, _ = digester.Write(blobContent) - digest := boshcrypto.NewDigest(boshcrypto.DigestAlgorithmSHA1, hex.EncodeToString(digester.Sum(nil))) - return compiledBlobstoreID, digest, nil - } -} - var _ = Describe("shippedPackages", func() { It("discovers packages from the job manifests in the log-cache release", func() { path := filepath.Join("testdata", "log-cache-release-3.0.9.tgz") @@ -782,210 +661,3 @@ var _ = Describe("shippedPackages", func() { Expect(err).To(MatchError(ContainSubstring("unknown package"))) }) }) - -func writePlainTarJobRelease(t GinkgoTInterface, release manifest.Manifest, jobMF string) string { - GinkgoHelper() - releaseMF, err := yaml.Marshal(release) - Expect(err).NotTo(HaveOccurred()) - inner := createTar(simpleFile("job.MF", []byte(jobMF), 0o644)) - archive, err := createTGZ(simpleFile("release.MF", releaseMF, 0o644), simpleFile("jobs/job.tgz", inner, 0o644)) - Expect(err).NotTo(HaveOccurred()) - path := filepath.Join(t.TempDir(), "release.tgz") - Expect(os.WriteFile(path, archive, 0o644)).To(Succeed()) - return path -} - -func writeRawJobRelease(t GinkgoTInterface, release manifest.Manifest, jobName string, inner []byte) string { - GinkgoHelper() - releaseMF, err := yaml.Marshal(release) - Expect(err).NotTo(HaveOccurred()) - archive, err := createTGZ(simpleFile("release.MF", releaseMF, 0o644), simpleFile("jobs/"+jobName+".tgz", inner, 0o644)) - Expect(err).NotTo(HaveOccurred()) - path := filepath.Join(t.TempDir(), "release.tgz") - Expect(os.WriteFile(path, archive, 0o644)).To(Succeed()) - return path -} - -func createTar(functions ...writeTarballFileFunc) []byte { - var buffer bytes.Buffer - tw := tar.NewWriter(&buffer) - for _, fn := range functions { - Expect(fn(tw)).To(Succeed()) - } - Expect(tw.Close()).To(Succeed()) - return buffer.Bytes() -} - -func writeShippedRelease(t GinkgoTInterface, release manifest.Manifest, jobs map[string]string, dotPrefix bool) string { - releaseMF, err := yaml.Marshal(release) - Expect(err).NotTo(HaveOccurred()) - files := []writeTarballFileFunc{simpleFile("release.MF", releaseMF, 0o644)} - for name, jobMF := range jobs { - innerName := "job.MF" - outerName := "jobs/" + name + ".tgz" - if dotPrefix { - innerName = "./" + innerName - outerName = "./" + outerName - } - var inner []byte - switch jobMF { - case "invalid-archive": - inner = []byte("invalid nested tar") - case "missing": - inner, err = createTGZ() - default: - inner, err = createTGZ(simpleFile(innerName, []byte(jobMF), 0o644)) - } - Expect(err).NotTo(HaveOccurred()) - files = append(files, simpleFile(outerName, inner, 0o644)) - } - archive, err := createTGZ(files...) - Expect(err).NotTo(HaveOccurred()) - path := filepath.Join(t.TempDir(), "release.tgz") - Expect(os.WriteFile(path, archive, 0o644)).To(Succeed()) - return path -} - -func mustReleaseManifest(path string) manifest.Manifest { - result, err := releasetarball.Manifest(path) - Expect(err).NotTo(HaveOccurred()) - return result -} - -func rewriteReleaseManifest(t GinkgoTInterface, source string, replacement manifest.Manifest) string { - GinkgoHelper() - f, err := os.Open(source) - Expect(err).NotTo(HaveOccurred()) - defer closeAndIgnoreError(f) - gr, err := gzip.NewReader(f) - Expect(err).NotTo(HaveOccurred()) - defer closeAndIgnoreError(gr) - tr := tar.NewReader(gr) - var files []writeTarballFileFunc - for { - h, err := tr.Next() - if errors.Is(err, io.EOF) { - break - } - Expect(err).NotTo(HaveOccurred()) - if h.FileInfo().IsDir() { - continue - } - contents, err := io.ReadAll(tr) - Expect(err).NotTo(HaveOccurred()) - if filepath.Base(h.Name) == "release.MF" { - contents, err = yaml.Marshal(replacement) - Expect(err).NotTo(HaveOccurred()) - } - files = append(files, simpleFile(h.Name, contents, h.Mode)) - } - archive, err := createTGZ(files...) - Expect(err).NotTo(HaveOccurred()) - path := filepath.Join(t.TempDir(), "release.tgz") - Expect(os.WriteFile(path, archive, 0o644)).To(Succeed()) - return path -} - -func rewriteJobPackages(t GinkgoTInterface, source string, packageNames []string) string { - GinkgoHelper() - f, err := os.Open(source) - Expect(err).NotTo(HaveOccurred()) - defer closeAndIgnoreError(f) - gr, err := gzip.NewReader(f) - Expect(err).NotTo(HaveOccurred()) - defer closeAndIgnoreError(gr) - tr := tar.NewReader(gr) - var files []writeTarballFileFunc - for { - h, err := tr.Next() - if errors.Is(err, io.EOF) { - break - } - Expect(err).NotTo(HaveOccurred()) - if h.FileInfo().IsDir() { - continue - } - contents, err := io.ReadAll(tr) - Expect(err).NotTo(HaveOccurred()) - if filepath.Base(filepath.Dir(h.Name)) == "jobs" && filepath.Ext(h.Name) == ".tgz" { - jobName := strings.TrimSuffix(filepath.Base(h.Name), ".tgz") - jobContents := fmt.Sprintf("name: %s\npackages: [%s]\n", jobName, strings.Join(packageNames, ", ")) - contents, err = createTGZ(simpleFile("job.MF", []byte(jobContents), 0o644)) - Expect(err).NotTo(HaveOccurred()) - } - files = append(files, simpleFile(h.Name, contents, h.Mode)) - } - archive, err := createTGZ(files...) - Expect(err).NotTo(HaveOccurred()) - path := filepath.Join(t.TempDir(), "release.tgz") - Expect(os.WriteFile(path, archive, 0o644)).To(Succeed()) - return path -} - -func removeJobArchives(t GinkgoTInterface, source string) string { - return rewriteJobArchives(t, source, true) -} - -func replaceJobArchives(t GinkgoTInterface, source string) string { - return rewriteJobArchives(t, source, false) -} - -func rewriteJobArchives(t GinkgoTInterface, source string, remove bool) string { - GinkgoHelper() - f, err := os.Open(source) - Expect(err).NotTo(HaveOccurred()) - defer closeAndIgnoreError(f) - gr, err := gzip.NewReader(f) - Expect(err).NotTo(HaveOccurred()) - defer closeAndIgnoreError(gr) - tr := tar.NewReader(gr) - var files []writeTarballFileFunc - for { - h, err := tr.Next() - if errors.Is(err, io.EOF) { - break - } - Expect(err).NotTo(HaveOccurred()) - if h.FileInfo().IsDir() { - continue - } - contents, err := io.ReadAll(tr) - Expect(err).NotTo(HaveOccurred()) - if filepath.Base(filepath.Dir(h.Name)) == "jobs" && filepath.Ext(h.Name) == ".tgz" { - if remove { - continue - } - contents, err = createTGZ() - Expect(err).NotTo(HaveOccurred()) - } - files = append(files, simpleFile(h.Name, contents, h.Mode)) - } - archive, err := createTGZ(files...) - Expect(err).NotTo(HaveOccurred()) - path := filepath.Join(t.TempDir(), "release.tgz") - Expect(os.WriteFile(path, archive, 0o644)).To(Succeed()) - return path -} - -func releaseManifestBytes(filePath string) ([]byte, error) { - f, err := os.Open(filePath) - if err != nil { - return nil, err - } - defer closeAndIgnoreError(f) - gr, err := gzip.NewReader(f) - if err != nil { - return nil, err - } - defer closeAndIgnoreError(gr) - tr := tar.NewReader(gr) - for { - h, err := tr.Next() - if err != nil { - return nil, err - } - if filepath.Base(h.Name) == "release.MF" { - return io.ReadAll(tr) - } - } -} diff --git a/releasetarball/export_test.go b/releasetarball/export_test.go index 6808a45a6..e08da35bd 100644 --- a/releasetarball/export_test.go +++ b/releasetarball/export_test.go @@ -1,16 +1,41 @@ package releasetarball import ( + "bytes" + "encoding/json" + "fmt" + "strings" + boshmodels "github.com/cloudfoundry/bosh-agent/v2/agent/applier/models" "github.com/cloudfoundry/bosh-cli/v7/release/manifest" ) +// These adapters expose private helpers only to the external test package. +// They are compiled by go test and are not part of the public API. type IdentityPkg = manifest.PackageRef type RewrittenIdentity = rewrittenPkg func DependencyKeyForTest(name string, all []IdentityPkg) (string, error) { - return dependencyKey(name, all) + packages := make(map[string]manifest.PackageRef, len(all)) + for _, p := range all { + packages[p.Name] = p + } + root, ok := packages[name] + if !ok { + return "", fmt.Errorf("unknown package %q", name) + } + tree, _, err := dependencyTree(root, packages, nil) + if err != nil { + return "", err + } + var output bytes.Buffer + encoder := json.NewEncoder(&output) + encoder.SetEscapeHTML(false) + if err := encoder.Encode(tree); err != nil { + return "", err + } + return strings.TrimSuffix(output.String(), "\n"), nil } func RewriteIdentitiesForTest(all []IdentityPkg, shipped map[string]bool) (map[string]RewrittenIdentity, error) { @@ -22,5 +47,6 @@ func ShippedPackagesForTest(path string, m manifest.Manifest) (map[string]bool, } func WriteCompiledReleaseForTest(m manifest.Manifest, outputDirectory, stemcellSlug, blobsDirectory, initialTarball string, sourcePackages []manifest.PackageRef, compiledPackages []boshmodels.Package, shipped map[string]bool) (string, error) { - return writeCompiledRelease(m, outputDirectory, stemcellSlug, blobsDirectory, initialTarball, sourcePackages, compiledPackages, CompileOptions{ExcludeCompileTimePackages: true}, shipped) + m.Packages = sourcePackages + return writeCompiledRelease(m, outputDirectory, stemcellSlug, blobsDirectory, initialTarball, compiledPackages, shipped) } diff --git a/releasetarball/fixtures_test.go b/releasetarball/fixtures_test.go index bf3be3a88..4ffeac025 100644 --- a/releasetarball/fixtures_test.go +++ b/releasetarball/fixtures_test.go @@ -27,72 +27,57 @@ import ( ) var _ = Describe("Compile fixtures", func() { - fixtureDir := os.Getenv("FIXTURES") - if fixtureDir == "" { - It("requires FIXTURES", func() { - Fail("FIXTURES required") - }) - return - } - - fixtures, err := filepath.Glob(filepath.Join(fixtureDir, "*.tgz")) - if err != nil { - It("reads FIXTURES", func() { Fail(err.Error()) }) - return - } - slices.Sort(fixtures) - if len(fixtures) == 0 { - It("requires fixture tarballs", func() { Fail("FIXTURES must contain at least one *.tgz file") }) - return - } - - for _, fixture := range fixtures { - fixture := fixture - It(filepath.Base(fixture), func() { - input := prepareFixture(GinkgoT(), fixture) - inputManifest := mustReleaseManifest(input) - shipped := jobPackageNames(GinkgoT(), input) - allPackages := packageNames(inputManifest.Packages) - kept := shipped - dropped := difference(allPackages, shipped) + DescribeTable("preserves compiled blobs and trims only build dependencies", func(name string) { + fixtureDir := os.Getenv("FIXTURES") + if fixtureDir == "" { + fixtureDir = "testdata" + } + fixture := filepath.Join(fixtureDir, name) + input := prepareFixture(GinkgoT(), fixture) + inputManifest := mustReleaseManifest(input) + shipped := jobPackageNames(GinkgoT(), input) + allPackages := packageNames(inputManifest.Packages) + kept := shipped + dropped := difference(allPackages, shipped) - workDir := GinkgoT().TempDir() - plainDir := filepath.Join(workDir, "plain") - flaggedDir := filepath.Join(workDir, "flagged") - blobs := directories.NewProvider(filepath.Join(workDir, "blobs")) - Expect(os.MkdirAll(blobs.BlobsDir(), 0o755)).To(Succeed()) - Expect(os.MkdirAll(plainDir, 0o755)).To(Succeed()) - Expect(os.MkdirAll(flaggedDir, 0o755)).To(Succeed()) - pkgCompiler := new(fakes.Compiler) - pkgCompiler.CompileCalls(fakeCompilation(blobs)) + workDir := GinkgoT().TempDir() + plainDir := filepath.Join(workDir, "plain") + flaggedDir := filepath.Join(workDir, "flagged") + blobs := directories.NewProvider(filepath.Join(workDir, "blobs")) + Expect(os.MkdirAll(blobs.BlobsDir(), 0o755)).To(Succeed()) + Expect(os.MkdirAll(plainDir, 0o755)).To(Succeed()) + Expect(os.MkdirAll(flaggedDir, 0o755)).To(Succeed()) + pkgCompiler := new(fakes.Compiler) + pkgCompiler.CompileCalls(fakeCompilation(blobs)) - plainPath, plain := compileFixture(pkgCompiler, input, blobs.BlobsDir(), plainDir, false) - flaggedPath, flagged := compileFixture(pkgCompiler, input, blobs.BlobsDir(), flaggedDir, true) - Expect(packageNamesCompiled(flagged.CompiledPkgs)).To(Equal(kept)) - Expect(packageNamesCompileTime(flagged.CompileTimePackages)).To(Equal(dropped)) - assertCompiledBlobs(GinkgoT(), flaggedPath, flagged.CompiledPkgs) - assertCompiledBlobs(GinkgoT(), plainPath, plain.CompiledPkgs) - assertNoDroppedBlobs(GinkgoT(), flaggedPath, dropped) - assertKeptPackages(inputManifest, flagged, kept) + plainPath, plain := compileFixture(pkgCompiler, input, blobs.BlobsDir(), plainDir, false) + flaggedPath, flagged := compileFixture(pkgCompiler, input, blobs.BlobsDir(), flaggedDir, true) + Expect(packageNamesCompiled(flagged.CompiledPkgs)).To(Equal(kept)) + Expect(packageNamesCompileTime(flagged.CompileTimePackages)).To(Equal(dropped)) + assertCompiledBlobs(flaggedPath, flagged.CompiledPkgs) + assertCompiledBlobs(plainPath, plain.CompiledPkgs) + assertKeptPackages(inputManifest, flagged, kept) - if len(dropped) == 0 { - plainBytes, err := os.ReadFile(plainPath) - Expect(err).NotTo(HaveOccurred()) - flaggedBytes, err := os.ReadFile(flaggedPath) - Expect(err).NotTo(HaveOccurred()) - Expect(sha256.Sum256(flaggedBytes)).To(Equal(sha256.Sum256(plainBytes))) - } else { - wantVersion := inputManifest.Version + "+trimmed" - if strings.Contains(inputManifest.Version, "+") { - wantVersion = inputManifest.Version + ".trimmed" - } - Expect(flagged.Version).To(Equal(wantVersion)) + if len(dropped) == 0 { + plainBytes, err := os.ReadFile(plainPath) + Expect(err).NotTo(HaveOccurred()) + flaggedBytes, err := os.ReadFile(flaggedPath) + Expect(err).NotTo(HaveOccurred()) + Expect(sha256.Sum256(flaggedBytes)).To(Equal(sha256.Sum256(plainBytes))) + } else { + wantVersion := inputManifest.Version + "+trimmed" + if strings.Contains(inputManifest.Version, "+") { + wantVersion = inputManifest.Version + ".trimmed" } + Expect(flagged.Version).To(Equal(wantVersion)) + } - copyFixtureOutputs(fixture, flaggedPath, plainPath) - GinkgoWriter.Printf("fixture %s: flagged=%s plain=%s kept=%d dropped=%v\n", filepath.Base(fixture), flaggedPath, plainPath, len(kept), dropped) - }) - } + copyFixtureOutputs(fixture, flaggedPath, plainPath) + GinkgoWriter.Printf("fixture %s: flagged=%s plain=%s kept=%d dropped=%v\n", filepath.Base(fixture), flaggedPath, plainPath, len(kept), dropped) + }, + Entry("source release", "log-cache-release-3.0.9.tgz"), + Entry("compiled release", "log-cache-3.0.9-banana-slug-1.23.tgz"), + ) }) func prepareFixture(t GinkgoTInterface, fixture string) string { @@ -210,7 +195,7 @@ func createTarballFromDir(t GinkgoTInterface, dir string) string { return output } -func assertCompiledBlobs(t GinkgoTInterface, tarball string, packages []manifest.CompiledPackageRef) { +func assertCompiledBlobs(tarball string, packages []manifest.CompiledPackageRef) { GinkgoHelper() f, err := os.Open(tarball) Expect(err).NotTo(HaveOccurred()) @@ -231,6 +216,17 @@ func assertCompiledBlobs(t GinkgoTInterface, tarball string, packages []manifest name := strings.TrimPrefix(h.Name, "./") entries[name] = append(entries[name], contents) } + var names []string + for name := range entries { + if strings.HasPrefix(name, "compiled_packages/") && strings.HasSuffix(name, ".tgz") { + names = append(names, name) + } + } + expected := make([]string, 0, len(packages)) + for _, pkg := range packages { + expected = append(expected, "compiled_packages/"+pkg.Name+".tgz") + } + Expect(names).To(ConsistOf(expected), "archive must contain exactly the manifest's compiled blobs") for _, pkg := range packages { name := "compiled_packages/" + pkg.Name + ".tgz" contents := entries[name] @@ -342,29 +338,6 @@ func dependencyClosureContainsDropped(name string, packages map[string]manifest. return false } -func assertNoDroppedBlobs(t GinkgoTInterface, tarball string, dropped []string) { - GinkgoHelper() - f, err := os.Open(tarball) - Expect(err).NotTo(HaveOccurred()) - defer closeAndIgnoreError(f) - gz, err := gzip.NewReader(f) - Expect(err).NotTo(HaveOccurred()) - defer closeAndIgnoreError(gz) - tr := tar.NewReader(gz) - entries := map[string]bool{} - for { - h, err := tr.Next() - if err == io.EOF { - break - } - Expect(err).NotTo(HaveOccurred()) - entries[h.Name] = true - } - for _, name := range dropped { - Expect(entries["compiled_packages/"+name+".tgz"]).To(BeFalse(), name) - } -} - func copyFixtureOutputs(fixture, flagged, plain string) { GinkgoHelper() outputDir := os.Getenv("FIXTURES_OUT") diff --git a/releasetarball/identity.go b/releasetarball/identity.go index e1ae1f2e4..723c134a6 100644 --- a/releasetarball/identity.go +++ b/releasetarball/identity.go @@ -4,7 +4,6 @@ import ( "bytes" "crypto/sha1" "crypto/sha256" - "encoding/hex" "encoding/json" "fmt" "slices" @@ -20,61 +19,32 @@ type rewrittenPkg struct { Rewritten bool } -func dependencyKey(name string, all []manifest.PackageRef) (string, error) { - packages := make(map[string]manifest.PackageRef, len(all)) - for _, p := range all { - packages[p.Name] = p - } - root, ok := packages[name] - if !ok { - return "", fmt.Errorf("unknown package %q", name) - } - - var arrayify func(manifest.PackageRef) ([]any, error) - arrayify = func(p manifest.PackageRef) ([]any, error) { - result := []any{p.Name, p.Version} - if len(p.Dependencies) > 0 { - dependencies := slices.Clone(p.Dependencies) - slices.Sort(dependencies) - children := make([]any, 0, len(dependencies)) - for _, dependency := range dependencies { - child, ok := packages[dependency] - if !ok { - return nil, fmt.Errorf("unknown dependency %q of package %q", dependency, p.Name) - } - childKey, err := arrayify(child) - if err != nil { - return nil, err - } - children = append(children, childKey) - } - result = append(result, children) - } - return result, nil - } - - dependencies := slices.Clone(root.Dependencies) +// dependencyTree follows the director's KeyGenerator#arrayify, including sorted +// children and omitted empty child arrays. Keep the director vectors in sync: +// https://github.com/cloudfoundry/bosh/blob/0410768ff620384100a277ff0683274970bc2c56/src/bosh-director/lib/bosh/director/key_generator.rb +// Compile checks for cycles before traversing this graph. +func dependencyTree(p manifest.PackageRef, packages map[string]manifest.PackageRef, shipped map[string]bool) ([]any, bool, error) { + dependencies := slices.Clone(p.Dependencies) slices.Sort(dependencies) - key := make([]any, 0, len(dependencies)) - for _, dependency := range dependencies { - p, ok := packages[dependency] + tree := make([]any, 0, len(dependencies)) + trimmed := false + for _, name := range dependencies { + child, ok := packages[name] if !ok { - return "", fmt.Errorf("unknown dependency %q of package %q", dependency, name) + return nil, false, fmt.Errorf("unknown dependency %q of package %q", name, p.Name) } - dependencyKey, err := arrayify(p) + children, childTrimmed, err := dependencyTree(child, packages, shipped) if err != nil { - return "", err + return nil, false, err } - key = append(key, dependencyKey) - } - - var output bytes.Buffer - encoder := json.NewEncoder(&output) - encoder.SetEscapeHTML(false) - if err := encoder.Encode(key); err != nil { - return "", err + entry := []any{child.Name, child.Version} + if len(children) > 0 { + entry = append(entry, children) + } + tree = append(tree, entry) + trimmed = trimmed || !shipped[name] || childTrimmed } - return strings.TrimSuffix(output.String(), "\n"), nil + return tree, trimmed, nil } func rewriteIdentities(all []manifest.PackageRef, shipped map[string]bool) (map[string]rewrittenPkg, error) { @@ -82,70 +52,48 @@ func rewriteIdentities(all []manifest.PackageRef, shipped map[string]bool) (map[ for _, p := range all { packages[p.Name] = p } - result := make(map[string]rewrittenPkg) for _, p := range all { if !shipped[p.Name] { continue } - key, err := dependencyKey(p.Name, all) + tree, trimmed, err := dependencyTree(p, packages, shipped) if err != nil { return nil, err } - - rewritten := false - var checkClosure func(manifest.PackageRef) error - checkClosure = func(current manifest.PackageRef) error { - for _, dependency := range current.Dependencies { - child, ok := packages[dependency] - if !ok { - return fmt.Errorf("unknown dependency %q of package %q", dependency, current.Name) - } - if !shipped[dependency] { - rewritten = true - } - if err := checkClosure(child); err != nil { - return err - } - } - return nil - } - if err := checkClosure(p); err != nil { - return nil, err - } - identity := rewrittenPkg{ Fingerprint: p.Fingerprint, OriginalFingerprint: p.Fingerprint, Dependencies: p.Dependencies, - Rewritten: rewritten, + Rewritten: trimmed, } - if rewritten { - var payload strings.Builder - payload.WriteString(p.Fingerprint) - payload.WriteByte('\n') - payload.WriteString(key) - payload.WriteByte('\n') - keptDependencies := make([]string, 0, len(p.Dependencies)) - for _, dependency := range p.Dependencies { - if shipped[dependency] { - keptDependencies = append(keptDependencies, dependency) + if trimmed { + identity.Dependencies = make([]string, 0, len(p.Dependencies)) + for _, name := range p.Dependencies { + if shipped[name] { + identity.Dependencies = append(identity.Dependencies, name) } } - slices.Sort(keptDependencies) - payload.WriteString(strings.Join(keptDependencies, ",")) - + slices.Sort(identity.Dependencies) + var key bytes.Buffer + encoder := json.NewEncoder(&key) + encoder.SetEscapeHTML(false) + if err := encoder.Encode(tree); err != nil { + return nil, err + } + // Preserve the existing trimmed identity format: source fingerprint, + // director dependency JSON, then comma-separated kept dependencies. + payload := p.Fingerprint + "\n" + strings.TrimSuffix(key.String(), "\n") + "\n" + strings.Join(identity.Dependencies, ",") switch len(p.Fingerprint) { case 40: - digest := sha1.Sum([]byte(payload.String())) - identity.Fingerprint = hex.EncodeToString(digest[:]) + digest := sha1.Sum([]byte(payload)) + identity.Fingerprint = fmt.Sprintf("%x", digest) case 64: - digest := sha256.Sum256([]byte(payload.String())) - identity.Fingerprint = hex.EncodeToString(digest[:]) + digest := sha256.Sum256([]byte(payload)) + identity.Fingerprint = fmt.Sprintf("%x", digest) default: return nil, fmt.Errorf("package %q has unsupported fingerprint length %d", p.Name, len(p.Fingerprint)) } - identity.Dependencies = keptDependencies } result[p.Name] = identity } diff --git a/releasetarball/identity_test.go b/releasetarball/identity_test.go index 65450d0db..282837fb5 100644 --- a/releasetarball/identity_test.go +++ b/releasetarball/identity_test.go @@ -336,29 +336,33 @@ func caseNamed(name string) identityCase { } var _ = Describe("package identities", func() { - for _, c := range identityCases { - It("computes identities for "+c.name, func() { - shipped := shippedSet(c.shipped) - result, err := releasetarball.RewriteIdentitiesForTest(c.packages, shipped) + DescribeTable("computes director-compatible identities", func(c identityCase) { + shipped := shippedSet(c.shipped) + result, err := releasetarball.RewriteIdentitiesForTest(c.packages, shipped) + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(HaveLen(len(c.expected))) + for name, expected := range c.expected { + actual, exists := result[name] + Expect(exists).To(BeTrue(), "package %q should be included", name) + key, err := releasetarball.DependencyKeyForTest(name, c.packages) Expect(err).NotTo(HaveOccurred()) - Expect(result).To(HaveLen(len(c.expected))) - for name, expected := range c.expected { - actual, exists := result[name] - Expect(exists).To(BeTrue(), "package %q should be included", name) - key, err := releasetarball.DependencyKeyForTest(name, c.packages) - Expect(err).NotTo(HaveOccurred()) - Expect(key).To(Equal(expected.dependencyKey), name) - Expect(actual.Rewritten).To(Equal(expected.rewritten), name) - Expect(actual.Fingerprint).To(Equal(expected.fingerprint), name) - Expect(actual.Dependencies).To(Equal(expected.dependencies), name) - } - for _, p := range c.packages { - if !shipped[p.Name] { - Expect(result).NotTo(HaveKey(p.Name), "dropped package %q should not be included", p.Name) - } + Expect(key).To(Equal(expected.dependencyKey), name) + Expect(actual.Rewritten).To(Equal(expected.rewritten), name) + Expect(actual.Fingerprint).To(Equal(expected.fingerprint), name) + Expect(actual.Dependencies).To(Equal(expected.dependencies), name) + } + for _, p := range c.packages { + if !shipped[p.Name] { + Expect(result).NotTo(HaveKey(p.Name), "dropped package %q should not be included", p.Name) } - }) - } + } + }, func() []TableEntry { + entries := make([]TableEntry, 0, len(identityCases)) + for _, c := range identityCases { + entries = append(entries, Entry(c.name, c)) + } + return entries + }()) It("keeps the fingerprint of a package that loses no dependencies", func() { c := caseNamed("leaf-nothing-dropped") diff --git a/releasetarball/shipped.go b/releasetarball/shipped.go index 5a1aec729..6ce33ab36 100644 --- a/releasetarball/shipped.go +++ b/releasetarball/shipped.go @@ -25,11 +25,7 @@ func shippedPackages(releaseTarballPath string, m releasemanifest.Manifest) (map for _, p := range m.CompiledPkgs { knownPackages[p.Name] = true } - type archivedJob struct { - name string - manifest manifest.Manifest - } - var archivedJobs []archivedJob + shipped := make(map[string]bool) if err := walkTarballFiles(releaseTarballPath, func(name string, _ *tar.Header, r io.Reader) (bool, error) { name = strings.TrimPrefix(name, "./") if !strings.HasPrefix(name, "jobs/") || !strings.HasSuffix(name, ".tgz") { @@ -43,7 +39,12 @@ func shippedPackages(releaseTarballPath string, m releasemanifest.Manifest) (map if _, declared := jobs[jobName]; declared { jobs[jobName] = true } - archivedJobs = append(archivedJobs, archivedJob{name: jobName, manifest: jobManifest}) + for _, packageName := range jobManifest.Packages { + if !knownPackages[packageName] { + return false, fmt.Errorf("job %q in release %q references unknown package %q", jobName, releaseTarballPath, packageName) + } + shipped[packageName] = true + } return true, nil }); err != nil { return nil, fmt.Errorf("failed to read release %q: %w", releaseTarballPath, err) @@ -54,15 +55,6 @@ func shippedPackages(releaseTarballPath string, m releasemanifest.Manifest) (map return nil, fmt.Errorf("release %q is missing job archive for job %q", releaseTarballPath, job.Name) } } - shipped := make(map[string]bool) - for _, job := range archivedJobs { - for _, packageName := range job.manifest.Packages { - if !knownPackages[packageName] { - return nil, fmt.Errorf("job %q in release %q references unknown package %q", job.name, releaseTarballPath, packageName) - } - shipped[packageName] = true - } - } if len(m.Packages) >= 1 && len(shipped) == 0 { return nil, fmt.Errorf("release %q has packages but no shipped packages in job.MF files", releaseTarballPath) } @@ -86,8 +78,6 @@ func readJobManifest(r io.Reader) (manifest.Manifest, error) { } tr := tar.NewReader(archive) var manifestContents []byte - manifestCount := 0 - manifestIsRegular := true for { h, err := tr.Next() if err != nil { @@ -99,27 +89,20 @@ func readJobManifest(r io.Reader) (manifest.Manifest, error) { if strings.TrimPrefix(h.Name, "./") != "job.MF" { continue } - manifestCount++ + if manifestContents != nil { + return manifest.Manifest{}, fmt.Errorf("job archive has duplicate job.MF entries") + } if h.Typeflag != tar.TypeReg { - manifestIsRegular = false - continue + return manifest.Manifest{}, fmt.Errorf("job archive job.MF is not a regular file") } - if manifestCount == 1 { - manifestContents, err = io.ReadAll(tr) - if err != nil { - return manifest.Manifest{}, err - } + manifestContents, err = io.ReadAll(tr) + if err != nil { + return manifest.Manifest{}, err } } - if manifestCount == 0 { + if manifestContents == nil { return manifest.Manifest{}, fmt.Errorf("job archive has no job.MF") } - if manifestCount > 1 { - return manifest.Manifest{}, fmt.Errorf("job archive has duplicate job.MF entries") - } - if !manifestIsRegular { - return manifest.Manifest{}, fmt.Errorf("job archive job.MF is not a regular file") - } var jobManifest manifest.Manifest if err := yaml.Unmarshal(manifestContents, &jobManifest); err != nil { return manifest.Manifest{}, err diff --git a/vendor/github.com/onsi/gomega/gstruct/elements.go b/vendor/github.com/onsi/gomega/gstruct/elements.go new file mode 100644 index 000000000..4a6114ab8 --- /dev/null +++ b/vendor/github.com/onsi/gomega/gstruct/elements.go @@ -0,0 +1,235 @@ +// untested sections: 6 + +package gstruct + +import ( + "errors" + "fmt" + "reflect" + "runtime/debug" + "strconv" + + "github.com/onsi/gomega/format" + errorsutil "github.com/onsi/gomega/gstruct/errors" + "github.com/onsi/gomega/types" +) + +// MatchAllElements succeeds if every element of a slice matches the element matcher it maps to +// through the id function, and every element matcher is matched. +// +// idFn := func(element any) string { +// return fmt.Sprintf("%v", element) +// } +// +// Expect([]string{"a", "b"}).To(MatchAllElements(idFn, Elements{ +// "a": Equal("a"), +// "b": Equal("b"), +// })) +func MatchAllElements(identifier Identifier, elements Elements) types.GomegaMatcher { + return &ElementsMatcher{ + Identifier: identifier, + Elements: elements, + } +} + +// MatchAllElementsWithIndex succeeds if every element of a slice matches the element matcher it maps to +// through the id with index function, and every element matcher is matched. +// +// idFn := func(index int, element any) string { +// return strconv.Itoa(index) +// } +// +// Expect([]string{"a", "b"}).To(MatchAllElements(idFn, Elements{ +// "0": Equal("a"), +// "1": Equal("b"), +// })) +func MatchAllElementsWithIndex(identifier IdentifierWithIndex, elements Elements) types.GomegaMatcher { + return &ElementsMatcher{ + Identifier: identifier, + Elements: elements, + } +} + +// MatchElements succeeds if each element of a slice matches the element matcher it maps to +// through the id function. It can ignore extra elements and/or missing elements. +// +// idFn := func(element any) string { +// return fmt.Sprintf("%v", element) +// } +// +// Expect([]string{"a", "b", "c"}).To(MatchElements(idFn, IgnoreExtras, Elements{ +// "a": Equal("a"), +// "b": Equal("b"), +// })) +// Expect([]string{"a", "c"}).To(MatchElements(idFn, IgnoreMissing, Elements{ +// "a": Equal("a"), +// "b": Equal("b"), +// "c": Equal("c"), +// "d": Equal("d"), +// })) +func MatchElements(identifier Identifier, options Options, elements Elements) types.GomegaMatcher { + return &ElementsMatcher{ + Identifier: identifier, + Elements: elements, + IgnoreExtras: options&IgnoreExtras != 0, + IgnoreMissing: options&IgnoreMissing != 0, + AllowDuplicates: options&AllowDuplicates != 0, + } +} + +// MatchElementsWithIndex succeeds if each element of a slice matches the element matcher it maps to +// through the id with index function. It can ignore extra elements and/or missing elements. +// +// idFn := func(index int, element any) string { +// return strconv.Itoa(index) +// } +// +// Expect([]string{"a", "b", "c"}).To(MatchElements(idFn, IgnoreExtras, Elements{ +// "0": Equal("a"), +// "1": Equal("b"), +// })) +// Expect([]string{"a", "c"}).To(MatchElements(idFn, IgnoreMissing, Elements{ +// "0": Equal("a"), +// "1": Equal("b"), +// "2": Equal("c"), +// "3": Equal("d"), +// })) +func MatchElementsWithIndex(identifier IdentifierWithIndex, options Options, elements Elements) types.GomegaMatcher { + return &ElementsMatcher{ + Identifier: identifier, + Elements: elements, + IgnoreExtras: options&IgnoreExtras != 0, + IgnoreMissing: options&IgnoreMissing != 0, + AllowDuplicates: options&AllowDuplicates != 0, + } +} + +// ElementsMatcher is a NestingMatcher that applies custom matchers to each element of a slice mapped +// by the Identifier function. +// TODO: Extend this to work with arrays & maps (map the key) as well. +type ElementsMatcher struct { + // Matchers for each element. + Elements Elements + // Function mapping an element to the string key identifying its matcher. + Identifier Identify + + // Whether to ignore extra elements or consider it an error. + IgnoreExtras bool + // Whether to ignore missing elements or consider it an error. + IgnoreMissing bool + // Whether to key duplicates when matching IDs. + AllowDuplicates bool + + // State. + failures []error +} + +// Element ID to matcher. +type Elements map[string]types.GomegaMatcher + +// Function for identifying (mapping) elements. +type Identifier func(element any) string + +// Calls the underlying function with the provided params. +// Identifier drops the index. +func (i Identifier) WithIndexAndElement(index int, element any) string { + return i(element) +} + +// Uses the index and element to generate an element name +type IdentifierWithIndex func(index int, element any) string + +// Calls the underlying function with the provided params. +// IdentifierWithIndex uses the index. +func (i IdentifierWithIndex) WithIndexAndElement(index int, element any) string { + return i(index, element) +} + +// Interface for identifying the element +type Identify interface { + WithIndexAndElement(i int, element any) string +} + +// IndexIdentity is a helper function for using an index as +// the key in the element map +func IndexIdentity(index int, _ any) string { + return strconv.Itoa(index) +} + +func (m *ElementsMatcher) Match(actual any) (success bool, err error) { + if reflect.TypeOf(actual).Kind() != reflect.Slice { + return false, fmt.Errorf("%v is type %T, expected slice", actual, actual) + } + + m.failures = m.matchElements(actual) + if len(m.failures) > 0 { + return false, nil + } + return true, nil +} + +func (m *ElementsMatcher) matchElements(actual any) (errs []error) { + // Provide more useful error messages in the case of a panic. + defer func() { + if err := recover(); err != nil { + errs = append(errs, fmt.Errorf("panic checking %+v: %v\n%s", actual, err, debug.Stack())) + } + }() + + val := reflect.ValueOf(actual) + elements := map[string]bool{} + for i := 0; i < val.Len(); i++ { + element := val.Index(i).Interface() + id := m.Identifier.WithIndexAndElement(i, element) + if elements[id] { + if !m.AllowDuplicates { + errs = append(errs, fmt.Errorf("found duplicate element ID %s", id)) + continue + } + } + elements[id] = true + + matcher, expected := m.Elements[id] + if !expected { + if !m.IgnoreExtras { + errs = append(errs, fmt.Errorf("unexpected element %s", id)) + } + continue + } + + match, err := matcher.Match(element) + if match { + continue + } + + if err == nil { + if nesting, ok := matcher.(errorsutil.NestingMatcher); ok { + err = errorsutil.AggregateError(nesting.Failures()) + } else { + err = errors.New(matcher.FailureMessage(element)) + } + } + errs = append(errs, errorsutil.Nest(fmt.Sprintf("[%s]", id), err)) + } + + for id := range m.Elements { + if !elements[id] && !m.IgnoreMissing { + errs = append(errs, fmt.Errorf("missing expected element %s", id)) + } + } + + return errs +} + +func (m *ElementsMatcher) FailureMessage(actual any) (message string) { + failure := errorsutil.AggregateError(m.failures) + return format.Message(actual, fmt.Sprintf("to match elements: %v", failure)) +} + +func (m *ElementsMatcher) NegatedFailureMessage(actual any) (message string) { + return format.Message(actual, "not to match elements") +} + +func (m *ElementsMatcher) Failures() []error { + return m.failures +} diff --git a/vendor/github.com/onsi/gomega/gstruct/errors/nested_types.go b/vendor/github.com/onsi/gomega/gstruct/errors/nested_types.go new file mode 100644 index 000000000..188492b21 --- /dev/null +++ b/vendor/github.com/onsi/gomega/gstruct/errors/nested_types.go @@ -0,0 +1,72 @@ +package errors + +import ( + "fmt" + "strings" + + "github.com/onsi/gomega/types" +) + +// A stateful matcher that nests other matchers within it and preserves the error types of the +// nested matcher failures. +type NestingMatcher interface { + types.GomegaMatcher + + // Returns the failures of nested matchers. + Failures() []error +} + +// An error type for labeling errors on deeply nested matchers. +type NestedError struct { + Path string + Err error +} + +func (e *NestedError) Error() string { + // Indent Errors. + indented := strings.Replace(e.Err.Error(), "\n", "\n\t", -1) + return fmt.Sprintf("%s:\n\t%v", e.Path, indented) +} + +// Create a NestedError with the given path. +// If err is a NestedError, prepend the path to it. +// If err is an AggregateError, recursively Nest each error. +func Nest(path string, err error) error { + if ag, ok := err.(AggregateError); ok { + var errs AggregateError + for _, e := range ag { + errs = append(errs, Nest(path, e)) + } + return errs + } + if ne, ok := err.(*NestedError); ok { + return &NestedError{ + Path: path + ne.Path, + Err: ne.Err, + } + } + return &NestedError{ + Path: path, + Err: err, + } +} + +// An error type for treating multiple errors as a single error. +type AggregateError []error + +// Error is part of the error interface. +func (err AggregateError) Error() string { + if len(err) == 0 { + // This should never happen, really. + return "" + } + if len(err) == 1 { + return err[0].Error() + } + result := fmt.Sprintf("[%s", err[0].Error()) + for i := 1; i < len(err); i++ { + result += fmt.Sprintf(", %s", err[i].Error()) + } + result += "]" + return result +} diff --git a/vendor/github.com/onsi/gomega/gstruct/fields.go b/vendor/github.com/onsi/gomega/gstruct/fields.go new file mode 100644 index 000000000..2f4685fc4 --- /dev/null +++ b/vendor/github.com/onsi/gomega/gstruct/fields.go @@ -0,0 +1,187 @@ +// untested sections: 6 + +package gstruct + +import ( + "errors" + "fmt" + "reflect" + "runtime/debug" + "strings" + "unicode" + + "github.com/onsi/gomega/format" + errorsutil "github.com/onsi/gomega/gstruct/errors" + "github.com/onsi/gomega/types" +) + +// MatchAllFields succeeds if every field of a struct matches the field matcher associated with +// it, and every element matcher is matched. +// +// actual := struct{ +// A int +// B []bool +// C string +// }{ +// A: 5, +// B: []bool{true, false}, +// C: "foo", +// } +// +// Expect(actual).To(MatchAllFields(Fields{ +// "A": Equal(5), +// "B": ConsistOf(true, false), +// "C": Equal("foo"), +// })) +func MatchAllFields(fields Fields) types.GomegaMatcher { + return &FieldsMatcher{ + Fields: fields, + } +} + +// MatchFields succeeds if each element of a struct matches the field matcher associated with +// it. It can ignore extra fields and/or missing fields. +// +// actual := struct{ +// A int +// B []bool +// C string +// }{ +// A: 5, +// B: []bool{true, false}, +// C: "foo", +// } +// +// Expect(actual).To(MatchFields(IgnoreExtras, Fields{ +// "A": Equal(5), +// "B": ConsistOf(true, false), +// })) +// Expect(actual).To(MatchFields(IgnoreMissing, Fields{ +// "A": Equal(5), +// "B": ConsistOf(true, false), +// "C": Equal("foo"), +// "D": Equal("extra"), +// })) +func MatchFields(options Options, fields Fields) types.GomegaMatcher { + return &FieldsMatcher{ + Fields: fields, + IgnoreExtras: options&IgnoreExtras != 0, + IgnoreUnexportedExtras: options&IgnoreUnexportedExtras != 0, + IgnoreMissing: options&IgnoreMissing != 0, + } +} + +type FieldsMatcher struct { + // Matchers for each field. + Fields Fields + + // Whether to ignore extra elements or consider it an error. + IgnoreExtras bool + // Whether to ignore unexported extra elements or consider it an error. + IgnoreUnexportedExtras bool + // Whether to ignore missing elements or consider it an error. + IgnoreMissing bool + + // State. + failures []error +} + +// Field name to matcher. +type Fields map[string]types.GomegaMatcher + +func (m *FieldsMatcher) Match(actual any) (success bool, err error) { + if reflect.TypeOf(actual).Kind() != reflect.Struct { + return false, fmt.Errorf("%v is type %T, expected struct", actual, actual) + } + + m.failures = m.matchFields(actual) + if len(m.failures) > 0 { + return false, nil + } + return true, nil +} + +func isExported(fieldName string) bool { + if fieldName == "" { + return false + } + r := []rune(fieldName)[0] + return unicode.IsUpper(r) +} + +func (m *FieldsMatcher) matchFields(actual any) (errs []error) { + val := reflect.ValueOf(actual) + typ := val.Type() + fields := map[string]bool{} + for i := 0; i < val.NumField(); i++ { + fieldName := typ.Field(i).Name + fields[fieldName] = true + + err := func() (err error) { + // This test relies heavily on reflect, which tends to panic. + // Recover here to provide more useful error messages in that case. + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("panic checking %+v: %v\n%s", actual, r, debug.Stack()) + } + }() + + matcher, expected := m.Fields[fieldName] + if !expected { + if m.IgnoreUnexportedExtras && !isExported(fieldName) { + return nil + } + if !m.IgnoreExtras { + return fmt.Errorf("unexpected field %s: %+v", fieldName, actual) + } + return nil + } + + var field any + if _, isIgnoreMatcher := matcher.(*IgnoreMatcher) ; isIgnoreMatcher { + field = struct {}{} // the matcher does not care about the actual value + } else { + field = val.Field(i).Interface() + } + + match, err := matcher.Match(field) + if err != nil { + return err + } else if !match { + if nesting, ok := matcher.(errorsutil.NestingMatcher); ok { + return errorsutil.AggregateError(nesting.Failures()) + } + return errors.New(matcher.FailureMessage(field)) + } + return nil + }() + if err != nil { + errs = append(errs, errorsutil.Nest("."+fieldName, err)) + } + } + + for field := range m.Fields { + if !fields[field] && !m.IgnoreMissing { + errs = append(errs, fmt.Errorf("missing expected field %s", field)) + } + } + + return errs +} + +func (m *FieldsMatcher) FailureMessage(actual any) (message string) { + failures := make([]string, len(m.failures)) + for i := range m.failures { + failures[i] = m.failures[i].Error() + } + return format.Message(reflect.TypeOf(actual).Name(), + fmt.Sprintf("to match fields: {\n%v\n}\n", strings.Join(failures, "\n"))) +} + +func (m *FieldsMatcher) NegatedFailureMessage(actual any) (message string) { + return format.Message(actual, "not to match fields") +} + +func (m *FieldsMatcher) Failures() []error { + return m.failures +} diff --git a/vendor/github.com/onsi/gomega/gstruct/ignore.go b/vendor/github.com/onsi/gomega/gstruct/ignore.go new file mode 100644 index 000000000..c8238d58f --- /dev/null +++ b/vendor/github.com/onsi/gomega/gstruct/ignore.go @@ -0,0 +1,41 @@ +// untested sections: 2 + +package gstruct + +import ( + "github.com/onsi/gomega/types" +) + +// Ignore ignores the actual value and always succeeds. +// +// Expect(nil).To(Ignore()) +// Expect(true).To(Ignore()) +func Ignore() types.GomegaMatcher { + return &IgnoreMatcher{true} +} + +// Reject ignores the actual value and always fails. It can be used in conjunction with IgnoreMissing +// to catch problematic elements, or to verify tests are running. +// +// Expect(nil).NotTo(Reject()) +// Expect(true).NotTo(Reject()) +func Reject() types.GomegaMatcher { + return &IgnoreMatcher{false} +} + +// A matcher that either always succeeds or always fails. +type IgnoreMatcher struct { + Succeed bool +} + +func (m *IgnoreMatcher) Match(actual any) (bool, error) { + return m.Succeed, nil +} + +func (m *IgnoreMatcher) FailureMessage(_ any) (message string) { + return "Unconditional failure" +} + +func (m *IgnoreMatcher) NegatedFailureMessage(_ any) (message string) { + return "Unconditional success" +} diff --git a/vendor/github.com/onsi/gomega/gstruct/keys.go b/vendor/github.com/onsi/gomega/gstruct/keys.go new file mode 100644 index 000000000..807a450b6 --- /dev/null +++ b/vendor/github.com/onsi/gomega/gstruct/keys.go @@ -0,0 +1,126 @@ +// untested sections: 6 + +package gstruct + +import ( + "errors" + "fmt" + "reflect" + "runtime/debug" + "strings" + + "github.com/onsi/gomega/format" + errorsutil "github.com/onsi/gomega/gstruct/errors" + "github.com/onsi/gomega/types" +) + +func MatchAllKeys(keys Keys) types.GomegaMatcher { + return &KeysMatcher{ + Keys: keys, + } +} + +func MatchKeys(options Options, keys Keys) types.GomegaMatcher { + return &KeysMatcher{ + Keys: keys, + IgnoreExtras: options&IgnoreExtras != 0, + IgnoreMissing: options&IgnoreMissing != 0, + } +} + +type KeysMatcher struct { + // Matchers for each key. + Keys Keys + + // Whether to ignore extra keys or consider it an error. + IgnoreExtras bool + // Whether to ignore missing keys or consider it an error. + IgnoreMissing bool + + // State. + failures []error +} + +type Keys map[any]types.GomegaMatcher + +func (m *KeysMatcher) Match(actual any) (success bool, err error) { + if reflect.TypeOf(actual).Kind() != reflect.Map { + return false, fmt.Errorf("%v is type %T, expected map", actual, actual) + } + + m.failures = m.matchKeys(actual) + if len(m.failures) > 0 { + return false, nil + } + return true, nil +} + +func (m *KeysMatcher) matchKeys(actual any) (errs []error) { + actualValue := reflect.ValueOf(actual) + keys := map[any]bool{} + for _, keyValue := range actualValue.MapKeys() { + key := keyValue.Interface() + keys[key] = true + + err := func() (err error) { + // This test relies heavily on reflect, which tends to panic. + // Recover here to provide more useful error messages in that case. + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("panic checking %+v: %v\n%s", actual, r, debug.Stack()) + } + }() + + matcher, ok := m.Keys[key] + if !ok { + if !m.IgnoreExtras { + return fmt.Errorf("unexpected key %s: %+v", key, actual) + } + return nil + } + + valInterface := actualValue.MapIndex(keyValue).Interface() + + match, err := matcher.Match(valInterface) + if err != nil { + return err + } + + if !match { + if nesting, ok := matcher.(errorsutil.NestingMatcher); ok { + return errorsutil.AggregateError(nesting.Failures()) + } + return errors.New(matcher.FailureMessage(valInterface)) + } + return nil + }() + if err != nil { + errs = append(errs, errorsutil.Nest(fmt.Sprintf(".%#v", key), err)) + } + } + + for key := range m.Keys { + if !keys[key] && !m.IgnoreMissing { + errs = append(errs, fmt.Errorf("missing expected key %s", key)) + } + } + + return errs +} + +func (m *KeysMatcher) FailureMessage(actual any) (message string) { + failures := make([]string, len(m.failures)) + for i := range m.failures { + failures[i] = m.failures[i].Error() + } + return format.Message(reflect.TypeOf(actual).Name(), + fmt.Sprintf("to match keys: {\n%v\n}\n", strings.Join(failures, "\n"))) +} + +func (m *KeysMatcher) NegatedFailureMessage(actual any) (message string) { + return format.Message(actual, "not to match keys") +} + +func (m *KeysMatcher) Failures() []error { + return m.failures +} diff --git a/vendor/github.com/onsi/gomega/gstruct/pointer.go b/vendor/github.com/onsi/gomega/gstruct/pointer.go new file mode 100644 index 000000000..54c9557fe --- /dev/null +++ b/vendor/github.com/onsi/gomega/gstruct/pointer.go @@ -0,0 +1,59 @@ +// untested sections: 3 + +package gstruct + +import ( + "fmt" + "reflect" + + "github.com/onsi/gomega/format" + "github.com/onsi/gomega/types" +) + +// PointTo applies the given matcher to the value pointed to by actual. It fails if the pointer is +// nil. +// +// actual := 5 +// Expect(&actual).To(PointTo(Equal(5))) +func PointTo(matcher types.GomegaMatcher) types.GomegaMatcher { + return &PointerMatcher{ + Matcher: matcher, + } +} + +type PointerMatcher struct { + Matcher types.GomegaMatcher + + // Failure message. + failure string +} + +func (m *PointerMatcher) Match(actual any) (bool, error) { + val := reflect.ValueOf(actual) + + // return error if actual type is not a pointer + if val.Kind() != reflect.Ptr { + return false, fmt.Errorf("PointerMatcher expects a pointer but we have '%s'", val.Kind()) + } + + if !val.IsValid() || val.IsNil() { + m.failure = format.Message(actual, "not to be ") + return false, nil + } + + // Forward the value. + elem := val.Elem().Interface() + match, err := m.Matcher.Match(elem) + if !match { + m.failure = m.Matcher.FailureMessage(elem) + } + return match, err +} + +func (m *PointerMatcher) FailureMessage(_ any) (message string) { + return m.failure +} + +func (m *PointerMatcher) NegatedFailureMessage(actual any) (message string) { + return m.Matcher.NegatedFailureMessage(actual) +} diff --git a/vendor/github.com/onsi/gomega/gstruct/types.go b/vendor/github.com/onsi/gomega/gstruct/types.go new file mode 100644 index 000000000..a5f6c390b --- /dev/null +++ b/vendor/github.com/onsi/gomega/gstruct/types.go @@ -0,0 +1,19 @@ +package gstruct + +// Options is the type for options passed to some matchers. +type Options int + +const ( + //IgnoreExtras tells the matcher to ignore extra elements or fields, rather than triggering a failure. + IgnoreExtras Options = 1 << iota + //IgnoreMissing tells the matcher to ignore missing elements or fields, rather than triggering a failure. + IgnoreMissing + //AllowDuplicates tells the matcher to permit multiple members of the slice to produce the same ID when + //considered by the identifier function. All members that map to a given key must still match successfully + //with the matcher that is provided for that key. + AllowDuplicates + //IgnoreUnexportedExtras tells the matcher to ignore extra unexported fields, rather than triggering a failure. + //it is not possible to check the value of unexported fields, so this option is only useful when you want to + //check every exported fields, but you don't care about extra unexported fields. + IgnoreUnexportedExtras +) diff --git a/vendor/modules.txt b/vendor/modules.txt index 4ac8061f7..e0973c951 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -303,6 +303,8 @@ github.com/onsi/gomega/format github.com/onsi/gomega/gbytes github.com/onsi/gomega/gexec github.com/onsi/gomega/ghttp +github.com/onsi/gomega/gstruct +github.com/onsi/gomega/gstruct/errors github.com/onsi/gomega/internal github.com/onsi/gomega/internal/gutil github.com/onsi/gomega/matchers From 46bae40c35734d586de0cba2c4f55bab839deeb9 Mon Sep 17 00:00:00 2001 From: Nishad Mathur Date: Wed, 28 Oct 2026 20:02:04 -0700 Subject: [PATCH 3/3] Preserve validation semantics and external fixture coverage Restore the original job archive validation path and explicit trimming option. Keep external FIXTURES discovery alongside the committed table entries, and name fingerprint payload components for readability. Add regression cases for empty manifests and validation order. The complete non-vendored production diff is 319 changed lines under the approved 320-line budget. --- releasetarball/compile.go | 8 +++--- releasetarball/compile_test.go | 37 ++++++++++++++++++++++++++ releasetarball/export_test.go | 2 +- releasetarball/fixtures_test.go | 27 ++++++++++++------- releasetarball/identity.go | 4 ++- releasetarball/shipped.go | 47 ++++++++++++++++++++++----------- 6 files changed, 95 insertions(+), 30 deletions(-) diff --git a/releasetarball/compile.go b/releasetarball/compile.go index 5d888cf4a..69114c191 100644 --- a/releasetarball/compile.go +++ b/releasetarball/compile.go @@ -117,7 +117,7 @@ func Compile(compiler boshcomp.Compiler, boshReleaseTarballPath, blobsDirectory, log.Printf("Finished packages compilation after %s", time.Since(start)) log.Printf("Archiving compiled BOSH Release %s/%s with stemcell %s", m.Name, m.Version, stemcellSlug) - return writeCompiledRelease(m, outputDirectory, stemcellSlug, blobsDirectory, boshReleaseTarballPath, compiledPackages, shipped) + return writeCompiledRelease(m, outputDirectory, stemcellSlug, blobsDirectory, boshReleaseTarballPath, compiledPackages, options, shipped) } func compilePackage(compiledPackages []boshmodels.Package, p manifest.PackageRef, blobstoreIDs map[string]string, compiler boshcomp.Compiler) ([]boshmodels.Package, error) { @@ -209,9 +209,9 @@ func newCompiledPackageRef(pkg boshmodels.Package, src manifest.PackageRef, stem return compiled } -func writeCompiledRelease(m manifest.Manifest, outputDirectory, stemcellFilenameSuffix, blobsDirectory, initialTarball string, compiledPackages []boshmodels.Package, shipped map[string]bool) (string, error) { +func writeCompiledRelease(m manifest.Manifest, outputDirectory, stemcellFilenameSuffix, blobsDirectory, initialTarball string, compiledPackages []boshmodels.Package, options CompileOptions, shipped map[string]bool) (string, error) { var identities map[string]rewrittenPkg - if shipped != nil { + if options.ExcludeCompileTimePackages { m.CompileTimePackages = nil var err error identities, err = rewriteIdentities(m.Packages, shipped) @@ -239,7 +239,7 @@ func writeCompiledRelease(m manifest.Manifest, outputDirectory, stemcellFilename keptCompiledPackages := make([]boshmodels.Package, 0, len(compiledPackages)) m.CompiledPkgs = make([]manifest.CompiledPackageRef, 0, len(compiledPackages)) for _, p := range compiledPackages { - if shipped != nil && !shipped[p.Name] { + if options.ExcludeCompileTimePackages && !shipped[p.Name] { continue } keptCompiledPackages = append(keptCompiledPackages, p) diff --git a/releasetarball/compile_test.go b/releasetarball/compile_test.go index 6595c7ee3..1e6d4ee97 100644 --- a/releasetarball/compile_test.go +++ b/releasetarball/compile_test.go @@ -619,6 +619,43 @@ var _ = Describe("shippedPackages", func() { } }) + DescribeTable("reports duplicate job.MF entries before validating their contents", func(kind byte, contents string) { + first := func(tw *tar.Writer) error { + if err := tw.WriteHeader(&tar.Header{Name: "job.MF", Typeflag: kind, Mode: 0o644, Size: int64(len(contents))}); err != nil { + return err + } + _, err := tw.Write([]byte(contents)) + return err + } + inner := createTar(first, simpleFile("./job.MF", []byte("packages: [pkg]\n"), 0o644)) + path := writeRawJobRelease(GinkgoT(), manifest.Manifest{ + Packages: []manifest.PackageRef{{Name: "pkg"}}, Jobs: []manifest.JobRef{{Name: "job"}}, + }, "job", inner) + _, err := releasetarball.ShippedPackagesForTest(path, mustReleaseManifest(path)) + Expect(err).To(MatchError(ContainSubstring("duplicate job.MF entries"))) + }, + Entry("empty first manifest", byte(tar.TypeReg), ""), + Entry("symlink first manifest", byte(tar.TypeSymlink), ""), + Entry("malformed first manifest", byte(tar.TypeReg), "packages: [unterminated"), + ) + + It("accepts an empty job.MF for a release without packages", func() { + path := writeShippedRelease(GinkgoT(), manifest.Manifest{ + Jobs: []manifest.JobRef{{Name: "job"}}, + }, map[string]string{"job": ""}, false) + shipped, err := releasetarball.ShippedPackagesForTest(path, mustReleaseManifest(path)) + Expect(err).NotTo(HaveOccurred()) + Expect(shipped).To(BeEmpty()) + }) + + It("reports missing job archives before validating referenced package names", func() { + path := writeShippedRelease(GinkgoT(), manifest.Manifest{ + Packages: []manifest.PackageRef{{Name: "pkg"}}, Jobs: []manifest.JobRef{{Name: "missing"}}, + }, map[string]string{"extra": "packages: [unknown]\n"}, false) + _, err := releasetarball.ShippedPackagesForTest(path, mustReleaseManifest(path)) + Expect(err).To(MatchError(ContainSubstring(`missing job archive for job "missing"`))) + }) + It("rejects a non-regular job.MF entry", func() { inner := createTar(func(tw *tar.Writer) error { return tw.WriteHeader(&tar.Header{Name: "job.MF", Mode: 0o777, Typeflag: tar.TypeSymlink, Linkname: "target"}) diff --git a/releasetarball/export_test.go b/releasetarball/export_test.go index e08da35bd..d18ea3e4f 100644 --- a/releasetarball/export_test.go +++ b/releasetarball/export_test.go @@ -48,5 +48,5 @@ func ShippedPackagesForTest(path string, m manifest.Manifest) (map[string]bool, func WriteCompiledReleaseForTest(m manifest.Manifest, outputDirectory, stemcellSlug, blobsDirectory, initialTarball string, sourcePackages []manifest.PackageRef, compiledPackages []boshmodels.Package, shipped map[string]bool) (string, error) { m.Packages = sourcePackages - return writeCompiledRelease(m, outputDirectory, stemcellSlug, blobsDirectory, initialTarball, compiledPackages, shipped) + return writeCompiledRelease(m, outputDirectory, stemcellSlug, blobsDirectory, initialTarball, compiledPackages, CompileOptions{ExcludeCompileTimePackages: true}, shipped) } diff --git a/releasetarball/fixtures_test.go b/releasetarball/fixtures_test.go index 4ffeac025..fbb00c8c5 100644 --- a/releasetarball/fixtures_test.go +++ b/releasetarball/fixtures_test.go @@ -27,12 +27,24 @@ import ( ) var _ = Describe("Compile fixtures", func() { - DescribeTable("preserves compiled blobs and trims only build dependencies", func(name string) { - fixtureDir := os.Getenv("FIXTURES") - if fixtureDir == "" { - fixtureDir = "testdata" + entries := []TableEntry{ + Entry("source release", filepath.Join("testdata", "log-cache-release-3.0.9.tgz")), + Entry("compiled release", filepath.Join("testdata", "log-cache-3.0.9-banana-slug-1.23.tgz")), + } + if fixtureDir := os.Getenv("FIXTURES"); fixtureDir != "" { + fixtures, err := filepath.Glob(filepath.Join(fixtureDir, "*.tgz")) + if err != nil || len(fixtures) == 0 { + It("requires readable fixture tarballs", func() { + Expect(err).NotTo(HaveOccurred()) + Expect(fixtures).NotTo(BeEmpty(), "FIXTURES must contain at least one *.tgz file") + }) + } else { + for _, fixture := range fixtures { + entries = append(entries, Entry(filepath.Base(fixture), fixture)) + } } - fixture := filepath.Join(fixtureDir, name) + } + DescribeTable("preserves compiled blobs and trims only build dependencies", func(fixture string) { input := prepareFixture(GinkgoT(), fixture) inputManifest := mustReleaseManifest(input) shipped := jobPackageNames(GinkgoT(), input) @@ -74,10 +86,7 @@ var _ = Describe("Compile fixtures", func() { copyFixtureOutputs(fixture, flaggedPath, plainPath) GinkgoWriter.Printf("fixture %s: flagged=%s plain=%s kept=%d dropped=%v\n", filepath.Base(fixture), flaggedPath, plainPath, len(kept), dropped) - }, - Entry("source release", "log-cache-release-3.0.9.tgz"), - Entry("compiled release", "log-cache-3.0.9-banana-slug-1.23.tgz"), - ) + }, entries) }) func prepareFixture(t GinkgoTInterface, fixture string) string { diff --git a/releasetarball/identity.go b/releasetarball/identity.go index 723c134a6..e4020c7d9 100644 --- a/releasetarball/identity.go +++ b/releasetarball/identity.go @@ -83,7 +83,9 @@ func rewriteIdentities(all []manifest.PackageRef, shipped map[string]bool) (map[ } // Preserve the existing trimmed identity format: source fingerprint, // director dependency JSON, then comma-separated kept dependencies. - payload := p.Fingerprint + "\n" + strings.TrimSuffix(key.String(), "\n") + "\n" + strings.Join(identity.Dependencies, ",") + dependencyKey := strings.TrimSuffix(key.String(), "\n") + keptDependencies := strings.Join(identity.Dependencies, ",") + payload := p.Fingerprint + "\n" + dependencyKey + "\n" + keptDependencies switch len(p.Fingerprint) { case 40: digest := sha1.Sum([]byte(payload)) diff --git a/releasetarball/shipped.go b/releasetarball/shipped.go index 6ce33ab36..5a1aec729 100644 --- a/releasetarball/shipped.go +++ b/releasetarball/shipped.go @@ -25,7 +25,11 @@ func shippedPackages(releaseTarballPath string, m releasemanifest.Manifest) (map for _, p := range m.CompiledPkgs { knownPackages[p.Name] = true } - shipped := make(map[string]bool) + type archivedJob struct { + name string + manifest manifest.Manifest + } + var archivedJobs []archivedJob if err := walkTarballFiles(releaseTarballPath, func(name string, _ *tar.Header, r io.Reader) (bool, error) { name = strings.TrimPrefix(name, "./") if !strings.HasPrefix(name, "jobs/") || !strings.HasSuffix(name, ".tgz") { @@ -39,12 +43,7 @@ func shippedPackages(releaseTarballPath string, m releasemanifest.Manifest) (map if _, declared := jobs[jobName]; declared { jobs[jobName] = true } - for _, packageName := range jobManifest.Packages { - if !knownPackages[packageName] { - return false, fmt.Errorf("job %q in release %q references unknown package %q", jobName, releaseTarballPath, packageName) - } - shipped[packageName] = true - } + archivedJobs = append(archivedJobs, archivedJob{name: jobName, manifest: jobManifest}) return true, nil }); err != nil { return nil, fmt.Errorf("failed to read release %q: %w", releaseTarballPath, err) @@ -55,6 +54,15 @@ func shippedPackages(releaseTarballPath string, m releasemanifest.Manifest) (map return nil, fmt.Errorf("release %q is missing job archive for job %q", releaseTarballPath, job.Name) } } + shipped := make(map[string]bool) + for _, job := range archivedJobs { + for _, packageName := range job.manifest.Packages { + if !knownPackages[packageName] { + return nil, fmt.Errorf("job %q in release %q references unknown package %q", job.name, releaseTarballPath, packageName) + } + shipped[packageName] = true + } + } if len(m.Packages) >= 1 && len(shipped) == 0 { return nil, fmt.Errorf("release %q has packages but no shipped packages in job.MF files", releaseTarballPath) } @@ -78,6 +86,8 @@ func readJobManifest(r io.Reader) (manifest.Manifest, error) { } tr := tar.NewReader(archive) var manifestContents []byte + manifestCount := 0 + manifestIsRegular := true for { h, err := tr.Next() if err != nil { @@ -89,20 +99,27 @@ func readJobManifest(r io.Reader) (manifest.Manifest, error) { if strings.TrimPrefix(h.Name, "./") != "job.MF" { continue } - if manifestContents != nil { - return manifest.Manifest{}, fmt.Errorf("job archive has duplicate job.MF entries") - } + manifestCount++ if h.Typeflag != tar.TypeReg { - return manifest.Manifest{}, fmt.Errorf("job archive job.MF is not a regular file") + manifestIsRegular = false + continue } - manifestContents, err = io.ReadAll(tr) - if err != nil { - return manifest.Manifest{}, err + if manifestCount == 1 { + manifestContents, err = io.ReadAll(tr) + if err != nil { + return manifest.Manifest{}, err + } } } - if manifestContents == nil { + if manifestCount == 0 { return manifest.Manifest{}, fmt.Errorf("job archive has no job.MF") } + if manifestCount > 1 { + return manifest.Manifest{}, fmt.Errorf("job archive has duplicate job.MF entries") + } + if !manifestIsRegular { + return manifest.Manifest{}, fmt.Errorf("job archive job.MF is not a regular file") + } var jobManifest manifest.Manifest if err := yaml.Unmarshal(manifestContents, &jobManifest); err != nil { return manifest.Manifest{}, err