diff --git a/main/compile.go b/main/compile.go index 0a6bbd6c8..7e825d525 100644 --- a/main/compile.go +++ b/main/compile.go @@ -41,7 +41,7 @@ func compileTarball(command string, args []string) { log.Fatal(err) } for _, releaseTarballPath := range options.SourceReleases { - compiledReleaseTarballPath, err := releasetarball.Compile(compiler, releaseTarballPath, dirProvider.BlobsDir(), options.OutputDirectory, compiledReleaseFileSuffix, releasetarball.CompileOptions{ExcludeCompileTimePackages: options.ExcludeCompileTimePackages}) + compiledReleaseTarballPath, err := releasetarball.Compile(compiler, releaseTarballPath, dirProvider.BlobsDir(), options.OutputDirectory, compiledReleaseFileSuffix, options.CompileOptions) if err != nil { log.Fatal(err) } @@ -50,9 +50,9 @@ func compileTarball(command string, args []string) { } type CompileTarballOptions struct { - OutputDirectory string - SourceReleases []string - ExcludeCompileTimePackages bool + OutputDirectory string + SourceReleases []string + releasetarball.CompileOptions } func newCompileTarballOptions(command string, args []string) (CompileTarballOptions, error) { diff --git a/releasetarball/archive_helpers_test.go b/releasetarball/archive_helpers_test.go new file mode 100644 index 000000000..63fc548e4 --- /dev/null +++ b/releasetarball/archive_helpers_test.go @@ -0,0 +1,286 @@ +package releasetarball_test + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "crypto/sha1" + "encoding/hex" + "errors" + "fmt" + "io" + "log" + "maps" + "os" + "path/filepath" + "strings" + "time" + + "github.com/cloudfoundry/bosh-agent/v2/agent/applier/models" + "github.com/cloudfoundry/bosh-agent/v2/agent/compiler" + "github.com/cloudfoundry/bosh-agent/v2/releasetarball" + "github.com/cloudfoundry/bosh-agent/v2/settings/directories" + "github.com/cloudfoundry/bosh-cli/v7/release/manifest" + boshcrypto "github.com/cloudfoundry/bosh-utils/crypto" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gopkg.in/yaml.v3" +) + +func closeAndIgnoreError(c io.Closer) { + _ = c.Close() //nolint:errcheck +} + +type writeTarballFileFunc func(tw *tar.Writer) error + +func createTGZ(functions ...writeTarballFileFunc) ([]byte, error) { + fn := func(w io.Writer) error { + gw := gzip.NewWriter(w) + defer closeAndIgnoreError(gw) + tw := tar.NewWriter(gw) + defer closeAndIgnoreError(tw) + for _, fn := range functions { + if err := fn(tw); err != nil { + return err + } + } + return nil + } + w := new(bytes.Buffer) + err := fn(w) + return w.Bytes(), err +} + +func simpleFile(name string, content []byte, mode int64) writeTarballFileFunc { + return func(tw *tar.Writer) error { + if err := tw.WriteHeader(newSimpleFileHeader(name, int64(len(content)), mode)); err != nil { + return err + } + _, err := tw.Write(content) + if err != nil { + return err + } + return nil + } +} + +func newSimpleFileHeader(name string, length, mode int64) *tar.Header { + t := time.Unix(0, 0).In(time.UTC) + return &tar.Header{ + Typeflag: tar.TypeReg, + Name: name, + Size: length, + ModTime: t, + AccessTime: t, + ChangeTime: t, + Mode: mode, + } +} + +func listFileNamesInTarball(t GinkgoTInterface, filePath string) []tar.Header { + f, err := os.Open(filePath) + if err != nil { + t.Fatal(err) + } + defer closeAndIgnoreError(f) + gr, err := gzip.NewReader(f) + if err != nil { + log.Fatal(err) + } + tr := tar.NewReader(gr) + var infos []tar.Header + for { + h, err := tr.Next() + if err != nil { + if errors.Is(err, io.EOF) { + break + } + t.Fatal(err) + } + cp := *h + cp.PAXRecords = maps.Clone(h.PAXRecords) + infos = append(infos, cp) + _, _ = io.Copy(io.Discard, tr) //nolint:errcheck + } + return infos +} + +func fakeCompilation(d directories.Provider) func(c compiler.Package, packages []models.Package) (string, boshcrypto.Digest, error) { + return func(c compiler.Package, packages []models.Package) (string, boshcrypto.Digest, error) { + blobContent, err := createTGZ(simpleFile("packaging", fmt.Appendf(nil, `"echo Compiled %q`, c.Name), 0o0744)) + if err != nil { + log.Fatal(err) + } + compiledBlobstoreID := fmt.Sprintf("%s-compiled-blob", c.Name) + if err := os.WriteFile(filepath.Join(d.BlobsDir(), compiledBlobstoreID), blobContent, 0o0644); err != nil { + log.Fatal(err) + } + digester := sha1.New() + _, _ = digester.Write(blobContent) + digest := boshcrypto.NewDigest(boshcrypto.DigestAlgorithmSHA1, hex.EncodeToString(digester.Sum(nil))) + return compiledBlobstoreID, digest, nil + } +} + +func writePlainTarJobRelease(t GinkgoTInterface, release manifest.Manifest, jobMF string) string { + return writeRawJobRelease(t, release, "job", createTar(simpleFile("job.MF", []byte(jobMF), 0o644))) +} + +func writeRawJobRelease(t GinkgoTInterface, release manifest.Manifest, jobName string, inner []byte) string { + GinkgoHelper() + releaseMF, err := yaml.Marshal(release) + Expect(err).NotTo(HaveOccurred()) + archive, err := createTGZ(simpleFile("release.MF", releaseMF, 0o644), simpleFile("jobs/"+jobName+".tgz", inner, 0o644)) + Expect(err).NotTo(HaveOccurred()) + path := filepath.Join(t.TempDir(), "release.tgz") + Expect(os.WriteFile(path, archive, 0o644)).To(Succeed()) + return path +} + +func createTar(functions ...writeTarballFileFunc) []byte { + var buffer bytes.Buffer + tw := tar.NewWriter(&buffer) + for _, fn := range functions { + Expect(fn(tw)).To(Succeed()) + } + Expect(tw.Close()).To(Succeed()) + return buffer.Bytes() +} + +func writeShippedRelease(t GinkgoTInterface, release manifest.Manifest, jobs map[string]string, dotPrefix bool) string { + releaseMF, err := yaml.Marshal(release) + Expect(err).NotTo(HaveOccurred()) + files := []writeTarballFileFunc{simpleFile("release.MF", releaseMF, 0o644)} + for name, jobMF := range jobs { + innerName := "job.MF" + outerName := "jobs/" + name + ".tgz" + if dotPrefix { + innerName = "./" + innerName + outerName = "./" + outerName + } + var inner []byte + switch jobMF { + case "invalid-archive": + inner = []byte("invalid nested tar") + case "missing": + inner, err = createTGZ() + default: + inner, err = createTGZ(simpleFile(innerName, []byte(jobMF), 0o644)) + } + Expect(err).NotTo(HaveOccurred()) + files = append(files, simpleFile(outerName, inner, 0o644)) + } + archive, err := createTGZ(files...) + Expect(err).NotTo(HaveOccurred()) + path := filepath.Join(t.TempDir(), "release.tgz") + Expect(os.WriteFile(path, archive, 0o644)).To(Succeed()) + return path +} + +func mustReleaseManifest(path string) manifest.Manifest { + result, err := releasetarball.Manifest(path) + Expect(err).NotTo(HaveOccurred()) + return result +} + +// rewriteReleaseArchive keeps archive setup in one place for malformed-input tests. +func rewriteReleaseArchive(t GinkgoTInterface, source string, rewrite func(string, []byte) ([]byte, bool)) string { + GinkgoHelper() + f, err := os.Open(source) + Expect(err).NotTo(HaveOccurred()) + defer closeAndIgnoreError(f) + gr, err := gzip.NewReader(f) + Expect(err).NotTo(HaveOccurred()) + defer closeAndIgnoreError(gr) + tr := tar.NewReader(gr) + var files []writeTarballFileFunc + for { + h, err := tr.Next() + if errors.Is(err, io.EOF) { + break + } + Expect(err).NotTo(HaveOccurred()) + if h.FileInfo().IsDir() { + continue + } + contents, err := io.ReadAll(tr) + Expect(err).NotTo(HaveOccurred()) + if contents, keep := rewrite(h.Name, contents); keep { + files = append(files, simpleFile(h.Name, contents, h.Mode)) + } + } + archive, err := createTGZ(files...) + Expect(err).NotTo(HaveOccurred()) + path := filepath.Join(t.TempDir(), "release.tgz") + Expect(os.WriteFile(path, archive, 0o644)).To(Succeed()) + return path +} + +func rewriteReleaseManifest(t GinkgoTInterface, source string, replacement manifest.Manifest) string { + return rewriteReleaseArchive(t, source, func(name string, contents []byte) ([]byte, bool) { + if filepath.Base(name) == "release.MF" { + var err error + contents, err = yaml.Marshal(replacement) + Expect(err).NotTo(HaveOccurred()) + } + return contents, true + }) +} + +func rewriteJobPackages(t GinkgoTInterface, source string, packageNames []string) string { + return rewriteReleaseArchive(t, source, func(name string, contents []byte) ([]byte, bool) { + if filepath.Base(filepath.Dir(name)) == "jobs" && filepath.Ext(name) == ".tgz" { + jobName := strings.TrimSuffix(filepath.Base(name), ".tgz") + jobMF := fmt.Sprintf("name: %s\npackages: [%s]\n", jobName, strings.Join(packageNames, ", ")) + var err error + contents, err = createTGZ(simpleFile("job.MF", []byte(jobMF), 0o644)) + Expect(err).NotTo(HaveOccurred()) + } + return contents, true + }) +} + +func removeJobArchives(t GinkgoTInterface, source string) string { + return rewriteJobArchives(t, source, true) +} + +func replaceJobArchives(t GinkgoTInterface, source string) string { + return rewriteJobArchives(t, source, false) +} + +func rewriteJobArchives(t GinkgoTInterface, source string, remove bool) string { + return rewriteReleaseArchive(t, source, func(name string, contents []byte) ([]byte, bool) { + if filepath.Base(filepath.Dir(name)) == "jobs" && filepath.Ext(name) == ".tgz" { + if remove { + return nil, false + } + var err error + contents, err = createTGZ() + Expect(err).NotTo(HaveOccurred()) + } + return contents, true + }) +} + +func releaseManifestBytes(filePath string) ([]byte, error) { + f, err := os.Open(filePath) + if err != nil { + return nil, err + } + defer closeAndIgnoreError(f) + gr, err := gzip.NewReader(f) + if err != nil { + return nil, err + } + defer closeAndIgnoreError(gr) + tr := tar.NewReader(gr) + for { + h, err := tr.Next() + if err != nil { + return nil, err + } + if filepath.Base(h.Name) == "release.MF" { + return io.ReadAll(tr) + } + } +} diff --git a/releasetarball/compile.go b/releasetarball/compile.go index d1c857110..69114c191 100644 --- a/releasetarball/compile.go +++ b/releasetarball/compile.go @@ -117,7 +117,7 @@ func Compile(compiler boshcomp.Compiler, boshReleaseTarballPath, blobsDirectory, log.Printf("Finished packages compilation after %s", time.Since(start)) log.Printf("Archiving compiled BOSH Release %s/%s with stemcell %s", m.Name, m.Version, stemcellSlug) - return writeCompiledRelease(m, outputDirectory, stemcellSlug, blobsDirectory, boshReleaseTarballPath, m.Packages, compiledPackages, options, shipped) + return writeCompiledRelease(m, outputDirectory, stemcellSlug, blobsDirectory, boshReleaseTarballPath, compiledPackages, options, shipped) } func compilePackage(compiledPackages []boshmodels.Package, p manifest.PackageRef, blobstoreIDs map[string]string, compiler boshcomp.Compiler) ([]boshmodels.Package, error) { @@ -209,16 +209,16 @@ func newCompiledPackageRef(pkg boshmodels.Package, src manifest.PackageRef, stem return compiled } -func writeCompiledRelease(m manifest.Manifest, outputDirectory, stemcellFilenameSuffix, blobsDirectory, initialTarball string, sourcePackages []manifest.PackageRef, compiledPackages []boshmodels.Package, options CompileOptions, shipped map[string]bool) (string, error) { - identities := map[string]rewrittenPkg{} +func writeCompiledRelease(m manifest.Manifest, outputDirectory, stemcellFilenameSuffix, blobsDirectory, initialTarball string, compiledPackages []boshmodels.Package, options CompileOptions, shipped map[string]bool) (string, error) { + var identities map[string]rewrittenPkg if options.ExcludeCompileTimePackages { m.CompileTimePackages = nil var err error - identities, err = rewriteIdentities(sourcePackages, shipped) + identities, err = rewriteIdentities(m.Packages, shipped) if err != nil { return "", err } - for _, p := range sourcePackages { + for _, p := range m.Packages { if shipped[p.Name] { continue } @@ -243,10 +243,10 @@ func writeCompiledRelease(m manifest.Manifest, outputDirectory, stemcellFilename continue } keptCompiledPackages = append(keptCompiledPackages, p) - srcIndex := slices.IndexFunc(sourcePackages, func(ref manifest.PackageRef) bool { + srcIndex := slices.IndexFunc(m.Packages, func(ref manifest.PackageRef) bool { return ref.Name == p.Name && ref.Version == p.Version }) - compiledRef := newCompiledPackageRef(p, sourcePackages[srcIndex], stemcellFilenameSuffix) + compiledRef := newCompiledPackageRef(p, m.Packages[srcIndex], stemcellFilenameSuffix) if identity := identities[p.Name]; identity.Rewritten { compiledRef.Version = identity.Fingerprint compiledRef.Fingerprint = identity.Fingerprint diff --git a/releasetarball/compile_test.go b/releasetarball/compile_test.go index a67a8d0cc..1e6d4ee97 100644 --- a/releasetarball/compile_test.go +++ b/releasetarball/compile_test.go @@ -4,28 +4,24 @@ import ( "archive/tar" "bytes" "compress/gzip" - "crypto/sha1" - "encoding/hex" "errors" "fmt" "io" "log" - "maps" "os" "path/filepath" "strings" "testing" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + . "github.com/onsi/gomega/gstruct" "github.com/cloudfoundry/bosh-cli/v7/release/manifest" boshcrypto "github.com/cloudfoundry/bosh-utils/crypto" "gopkg.in/yaml.v3" "github.com/cloudfoundry/bosh-agent/v2/agent/applier/models" - "github.com/cloudfoundry/bosh-agent/v2/agent/compiler" "github.com/cloudfoundry/bosh-agent/v2/releasetarball" "github.com/cloudfoundry/bosh-agent/v2/releasetarball/internal/fakes" "github.com/cloudfoundry/bosh-agent/v2/settings/directories" @@ -136,11 +132,12 @@ var _ = Describe("Compile", func() { HaveField("Name", "log-cache"), HaveField("Name", "log-cache-cf-auth-proxy"), HaveField("Name", "log-cache-gateway"), HaveField("Name", "log-cache-syslog-server"), )) - Expect(compiled.CompileTimePackages).To(HaveLen(1)) - Expect(compiled.CompileTimePackages[0].Name).To(Equal(source.Packages[0].Name)) - Expect(compiled.CompileTimePackages[0].Version).To(Equal(source.Packages[0].Version)) - Expect(compiled.CompileTimePackages[0].Fingerprint).To(Equal(source.Packages[0].Fingerprint)) - Expect(compiled.CompileTimePackages[0].Dependencies).To(Equal(source.Packages[0].Dependencies)) + Expect(compiled.CompileTimePackages).To(ConsistOf(MatchAllFields(Fields{ + "Name": Equal(source.Packages[0].Name), + "Version": Equal(source.Packages[0].Version), + "Fingerprint": Equal(source.Packages[0].Fingerprint), + "Dependencies": Equal(source.Packages[0].Dependencies), + }))) rawManifest, err := releaseManifestBytes(resultPath) Expect(err).NotTo(HaveOccurred()) if capturePath := os.Getenv("CAPTURE_COMPILE_MANIFEST"); capturePath != "" { @@ -301,42 +298,19 @@ var _ = Describe("Compile", func() { Expect(entries).To(BeEmpty()) }) - It("fails before compilation or output for missing job archives, missing job.MF, and an empty shipped set", func() { - fixture := filepath.Join("testdata", "log-cache-release-3.0.9.tgz") - tests := []struct { - name string - input func() string - want string - }{ - { - name: "missing archive", - input: func() string { return removeJobArchives(GinkgoT(), fixture) }, - want: "missing job archive", - }, - { - name: "missing job.MF", - input: func() string { return replaceJobArchives(GinkgoT(), fixture) }, - want: "no job.MF", - }, - { - name: "empty shipped set", - input: func() string { return rewriteJobPackages(GinkgoT(), fixture, nil) }, - want: "no shipped packages", - }, - } - for _, tt := range tests { - By(tt.name) - output := GinkgoT().TempDir() - pkgCompiler = new(fakes.Compiler) - pkgCompiler.CompileCalls(fakeCompilation(d)) - _, err := releasetarball.Compile(pkgCompiler, tt.input(), d.BlobsDir(), output, stemcellSlug, releasetarball.CompileOptions{ExcludeCompileTimePackages: true}) - Expect(err).To(MatchError(ContainSubstring(tt.want))) - Expect(pkgCompiler.CompileCallCount()).To(BeZero()) - entries, err := os.ReadDir(output) - Expect(err).NotTo(HaveOccurred()) - Expect(entries).To(BeEmpty()) - } - }) + DescribeTable("fails before compilation or output for invalid job archives", func(rewrite func(GinkgoTInterface, string) string, want string) { + input := rewrite(GinkgoT(), filepath.Join("testdata", "log-cache-release-3.0.9.tgz")) + _, err := releasetarball.Compile(pkgCompiler, input, d.BlobsDir(), releasesOutputDir, stemcellSlug, releasetarball.CompileOptions{ExcludeCompileTimePackages: true}) + Expect(err).To(MatchError(ContainSubstring(want))) + Expect(pkgCompiler.CompileCallCount()).To(BeZero()) + Expect(os.ReadDir(releasesOutputDir)).To(BeEmpty()) + }, + Entry("missing archive", removeJobArchives, "missing job archive"), + Entry("missing job.MF", replaceJobArchives, "no job.MF"), + Entry("empty shipped set", func(t GinkgoTInterface, input string) string { + return rewriteJobPackages(t, input, nil) + }, "no shipped packages"), + ) When("compiling a tarball with a package dependency cycle", func() { BeforeEach(func() { @@ -523,101 +497,6 @@ func assertPackageFields(compiledManifest, sourceManifest manifest.Manifest, ind Expect(compiledManifest.CompiledPkgs[index].Version).To(Equal(compiledManifest.CompiledPkgs[index].Fingerprint), "this seems to be equal on releases exported from a director") } -func closeAndIgnoreError(c io.Closer) { - _ = c.Close() //nolint:errcheck -} - -type writeTarballFileFunc func(tw *tar.Writer) error - -func createTGZ(functions ...writeTarballFileFunc) ([]byte, error) { - fn := func(w io.Writer) error { - gw := gzip.NewWriter(w) - defer closeAndIgnoreError(gw) - tw := tar.NewWriter(gw) - defer closeAndIgnoreError(tw) - for _, fn := range functions { - if err := fn(tw); err != nil { - return err - } - } - return nil - } - w := new(bytes.Buffer) - err := fn(w) - return w.Bytes(), err -} - -func simpleFile(name string, content []byte, mode int64) writeTarballFileFunc { - return func(tw *tar.Writer) error { - if err := tw.WriteHeader(newSimpleFileHeader(name, int64(len(content)), mode)); err != nil { - return err - } - _, err := tw.Write(content) - if err != nil { - return err - } - return nil - } -} - -func newSimpleFileHeader(name string, length, mode int64) *tar.Header { - t := time.Unix(0, 0).In(time.UTC) - return &tar.Header{ - Typeflag: tar.TypeReg, - Name: name, - Size: length, - ModTime: t, - AccessTime: t, - ChangeTime: t, - Mode: mode, - } -} - -func listFileNamesInTarball(t GinkgoTInterface, filePath string) []tar.Header { - f, err := os.Open(filePath) - if err != nil { - t.Fatal(err) - } - defer closeAndIgnoreError(f) - gr, err := gzip.NewReader(f) - if err != nil { - log.Fatal(err) - } - tr := tar.NewReader(gr) - var infos []tar.Header - for { - h, err := tr.Next() - if err != nil { - if errors.Is(err, io.EOF) { - break - } - t.Fatal(err) - } - cp := *h - cp.PAXRecords = maps.Clone(h.PAXRecords) - infos = append(infos, cp) - _, _ = io.Copy(io.Discard, tr) //nolint:errcheck - } - return infos -} - -func fakeCompilation(d directories.Provider) func(c compiler.Package, packages []models.Package) (string, boshcrypto.Digest, error) { - return func(c compiler.Package, packages []models.Package) (string, boshcrypto.Digest, error) { - blobContent, err := createTGZ(simpleFile("packaging", fmt.Appendf(nil, `"echo Compiled %q`, c.Name), 0o0744)) - if err != nil { - log.Fatal(err) - } - compiledBlobstoreID := fmt.Sprintf("%s-compiled-blob", c.Name) - if err := os.WriteFile(filepath.Join(d.BlobsDir(), compiledBlobstoreID), blobContent, 0o0644); err != nil { - log.Fatal(err) - } - digester := sha1.New() - _, _ = digester.Write(blobContent) - digest := boshcrypto.NewDigest(boshcrypto.DigestAlgorithmSHA1, hex.EncodeToString(digester.Sum(nil))) - return compiledBlobstoreID, digest, nil - } -} - var _ = Describe("shippedPackages", func() { It("discovers packages from the job manifests in the log-cache release", func() { path := filepath.Join("testdata", "log-cache-release-3.0.9.tgz") @@ -740,6 +619,43 @@ var _ = Describe("shippedPackages", func() { } }) + DescribeTable("reports duplicate job.MF entries before validating their contents", func(kind byte, contents string) { + first := func(tw *tar.Writer) error { + if err := tw.WriteHeader(&tar.Header{Name: "job.MF", Typeflag: kind, Mode: 0o644, Size: int64(len(contents))}); err != nil { + return err + } + _, err := tw.Write([]byte(contents)) + return err + } + inner := createTar(first, simpleFile("./job.MF", []byte("packages: [pkg]\n"), 0o644)) + path := writeRawJobRelease(GinkgoT(), manifest.Manifest{ + Packages: []manifest.PackageRef{{Name: "pkg"}}, Jobs: []manifest.JobRef{{Name: "job"}}, + }, "job", inner) + _, err := releasetarball.ShippedPackagesForTest(path, mustReleaseManifest(path)) + Expect(err).To(MatchError(ContainSubstring("duplicate job.MF entries"))) + }, + Entry("empty first manifest", byte(tar.TypeReg), ""), + Entry("symlink first manifest", byte(tar.TypeSymlink), ""), + Entry("malformed first manifest", byte(tar.TypeReg), "packages: [unterminated"), + ) + + It("accepts an empty job.MF for a release without packages", func() { + path := writeShippedRelease(GinkgoT(), manifest.Manifest{ + Jobs: []manifest.JobRef{{Name: "job"}}, + }, map[string]string{"job": ""}, false) + shipped, err := releasetarball.ShippedPackagesForTest(path, mustReleaseManifest(path)) + Expect(err).NotTo(HaveOccurred()) + Expect(shipped).To(BeEmpty()) + }) + + It("reports missing job archives before validating referenced package names", func() { + path := writeShippedRelease(GinkgoT(), manifest.Manifest{ + Packages: []manifest.PackageRef{{Name: "pkg"}}, Jobs: []manifest.JobRef{{Name: "missing"}}, + }, map[string]string{"extra": "packages: [unknown]\n"}, false) + _, err := releasetarball.ShippedPackagesForTest(path, mustReleaseManifest(path)) + Expect(err).To(MatchError(ContainSubstring(`missing job archive for job "missing"`))) + }) + It("rejects a non-regular job.MF entry", func() { inner := createTar(func(tw *tar.Writer) error { return tw.WriteHeader(&tar.Header{Name: "job.MF", Mode: 0o777, Typeflag: tar.TypeSymlink, Linkname: "target"}) @@ -782,210 +698,3 @@ var _ = Describe("shippedPackages", func() { Expect(err).To(MatchError(ContainSubstring("unknown package"))) }) }) - -func writePlainTarJobRelease(t GinkgoTInterface, release manifest.Manifest, jobMF string) string { - GinkgoHelper() - releaseMF, err := yaml.Marshal(release) - Expect(err).NotTo(HaveOccurred()) - inner := createTar(simpleFile("job.MF", []byte(jobMF), 0o644)) - archive, err := createTGZ(simpleFile("release.MF", releaseMF, 0o644), simpleFile("jobs/job.tgz", inner, 0o644)) - Expect(err).NotTo(HaveOccurred()) - path := filepath.Join(t.TempDir(), "release.tgz") - Expect(os.WriteFile(path, archive, 0o644)).To(Succeed()) - return path -} - -func writeRawJobRelease(t GinkgoTInterface, release manifest.Manifest, jobName string, inner []byte) string { - GinkgoHelper() - releaseMF, err := yaml.Marshal(release) - Expect(err).NotTo(HaveOccurred()) - archive, err := createTGZ(simpleFile("release.MF", releaseMF, 0o644), simpleFile("jobs/"+jobName+".tgz", inner, 0o644)) - Expect(err).NotTo(HaveOccurred()) - path := filepath.Join(t.TempDir(), "release.tgz") - Expect(os.WriteFile(path, archive, 0o644)).To(Succeed()) - return path -} - -func createTar(functions ...writeTarballFileFunc) []byte { - var buffer bytes.Buffer - tw := tar.NewWriter(&buffer) - for _, fn := range functions { - Expect(fn(tw)).To(Succeed()) - } - Expect(tw.Close()).To(Succeed()) - return buffer.Bytes() -} - -func writeShippedRelease(t GinkgoTInterface, release manifest.Manifest, jobs map[string]string, dotPrefix bool) string { - releaseMF, err := yaml.Marshal(release) - Expect(err).NotTo(HaveOccurred()) - files := []writeTarballFileFunc{simpleFile("release.MF", releaseMF, 0o644)} - for name, jobMF := range jobs { - innerName := "job.MF" - outerName := "jobs/" + name + ".tgz" - if dotPrefix { - innerName = "./" + innerName - outerName = "./" + outerName - } - var inner []byte - switch jobMF { - case "invalid-archive": - inner = []byte("invalid nested tar") - case "missing": - inner, err = createTGZ() - default: - inner, err = createTGZ(simpleFile(innerName, []byte(jobMF), 0o644)) - } - Expect(err).NotTo(HaveOccurred()) - files = append(files, simpleFile(outerName, inner, 0o644)) - } - archive, err := createTGZ(files...) - Expect(err).NotTo(HaveOccurred()) - path := filepath.Join(t.TempDir(), "release.tgz") - Expect(os.WriteFile(path, archive, 0o644)).To(Succeed()) - return path -} - -func mustReleaseManifest(path string) manifest.Manifest { - result, err := releasetarball.Manifest(path) - Expect(err).NotTo(HaveOccurred()) - return result -} - -func rewriteReleaseManifest(t GinkgoTInterface, source string, replacement manifest.Manifest) string { - GinkgoHelper() - f, err := os.Open(source) - Expect(err).NotTo(HaveOccurred()) - defer closeAndIgnoreError(f) - gr, err := gzip.NewReader(f) - Expect(err).NotTo(HaveOccurred()) - defer closeAndIgnoreError(gr) - tr := tar.NewReader(gr) - var files []writeTarballFileFunc - for { - h, err := tr.Next() - if errors.Is(err, io.EOF) { - break - } - Expect(err).NotTo(HaveOccurred()) - if h.FileInfo().IsDir() { - continue - } - contents, err := io.ReadAll(tr) - Expect(err).NotTo(HaveOccurred()) - if filepath.Base(h.Name) == "release.MF" { - contents, err = yaml.Marshal(replacement) - Expect(err).NotTo(HaveOccurred()) - } - files = append(files, simpleFile(h.Name, contents, h.Mode)) - } - archive, err := createTGZ(files...) - Expect(err).NotTo(HaveOccurred()) - path := filepath.Join(t.TempDir(), "release.tgz") - Expect(os.WriteFile(path, archive, 0o644)).To(Succeed()) - return path -} - -func rewriteJobPackages(t GinkgoTInterface, source string, packageNames []string) string { - GinkgoHelper() - f, err := os.Open(source) - Expect(err).NotTo(HaveOccurred()) - defer closeAndIgnoreError(f) - gr, err := gzip.NewReader(f) - Expect(err).NotTo(HaveOccurred()) - defer closeAndIgnoreError(gr) - tr := tar.NewReader(gr) - var files []writeTarballFileFunc - for { - h, err := tr.Next() - if errors.Is(err, io.EOF) { - break - } - Expect(err).NotTo(HaveOccurred()) - if h.FileInfo().IsDir() { - continue - } - contents, err := io.ReadAll(tr) - Expect(err).NotTo(HaveOccurred()) - if filepath.Base(filepath.Dir(h.Name)) == "jobs" && filepath.Ext(h.Name) == ".tgz" { - jobName := strings.TrimSuffix(filepath.Base(h.Name), ".tgz") - jobContents := fmt.Sprintf("name: %s\npackages: [%s]\n", jobName, strings.Join(packageNames, ", ")) - contents, err = createTGZ(simpleFile("job.MF", []byte(jobContents), 0o644)) - Expect(err).NotTo(HaveOccurred()) - } - files = append(files, simpleFile(h.Name, contents, h.Mode)) - } - archive, err := createTGZ(files...) - Expect(err).NotTo(HaveOccurred()) - path := filepath.Join(t.TempDir(), "release.tgz") - Expect(os.WriteFile(path, archive, 0o644)).To(Succeed()) - return path -} - -func removeJobArchives(t GinkgoTInterface, source string) string { - return rewriteJobArchives(t, source, true) -} - -func replaceJobArchives(t GinkgoTInterface, source string) string { - return rewriteJobArchives(t, source, false) -} - -func rewriteJobArchives(t GinkgoTInterface, source string, remove bool) string { - GinkgoHelper() - f, err := os.Open(source) - Expect(err).NotTo(HaveOccurred()) - defer closeAndIgnoreError(f) - gr, err := gzip.NewReader(f) - Expect(err).NotTo(HaveOccurred()) - defer closeAndIgnoreError(gr) - tr := tar.NewReader(gr) - var files []writeTarballFileFunc - for { - h, err := tr.Next() - if errors.Is(err, io.EOF) { - break - } - Expect(err).NotTo(HaveOccurred()) - if h.FileInfo().IsDir() { - continue - } - contents, err := io.ReadAll(tr) - Expect(err).NotTo(HaveOccurred()) - if filepath.Base(filepath.Dir(h.Name)) == "jobs" && filepath.Ext(h.Name) == ".tgz" { - if remove { - continue - } - contents, err = createTGZ() - Expect(err).NotTo(HaveOccurred()) - } - files = append(files, simpleFile(h.Name, contents, h.Mode)) - } - archive, err := createTGZ(files...) - Expect(err).NotTo(HaveOccurred()) - path := filepath.Join(t.TempDir(), "release.tgz") - Expect(os.WriteFile(path, archive, 0o644)).To(Succeed()) - return path -} - -func releaseManifestBytes(filePath string) ([]byte, error) { - f, err := os.Open(filePath) - if err != nil { - return nil, err - } - defer closeAndIgnoreError(f) - gr, err := gzip.NewReader(f) - if err != nil { - return nil, err - } - defer closeAndIgnoreError(gr) - tr := tar.NewReader(gr) - for { - h, err := tr.Next() - if err != nil { - return nil, err - } - if filepath.Base(h.Name) == "release.MF" { - return io.ReadAll(tr) - } - } -} diff --git a/releasetarball/export_test.go b/releasetarball/export_test.go index 6808a45a6..d18ea3e4f 100644 --- a/releasetarball/export_test.go +++ b/releasetarball/export_test.go @@ -1,16 +1,41 @@ package releasetarball import ( + "bytes" + "encoding/json" + "fmt" + "strings" + boshmodels "github.com/cloudfoundry/bosh-agent/v2/agent/applier/models" "github.com/cloudfoundry/bosh-cli/v7/release/manifest" ) +// These adapters expose private helpers only to the external test package. +// They are compiled by go test and are not part of the public API. type IdentityPkg = manifest.PackageRef type RewrittenIdentity = rewrittenPkg func DependencyKeyForTest(name string, all []IdentityPkg) (string, error) { - return dependencyKey(name, all) + packages := make(map[string]manifest.PackageRef, len(all)) + for _, p := range all { + packages[p.Name] = p + } + root, ok := packages[name] + if !ok { + return "", fmt.Errorf("unknown package %q", name) + } + tree, _, err := dependencyTree(root, packages, nil) + if err != nil { + return "", err + } + var output bytes.Buffer + encoder := json.NewEncoder(&output) + encoder.SetEscapeHTML(false) + if err := encoder.Encode(tree); err != nil { + return "", err + } + return strings.TrimSuffix(output.String(), "\n"), nil } func RewriteIdentitiesForTest(all []IdentityPkg, shipped map[string]bool) (map[string]RewrittenIdentity, error) { @@ -22,5 +47,6 @@ func ShippedPackagesForTest(path string, m manifest.Manifest) (map[string]bool, } func WriteCompiledReleaseForTest(m manifest.Manifest, outputDirectory, stemcellSlug, blobsDirectory, initialTarball string, sourcePackages []manifest.PackageRef, compiledPackages []boshmodels.Package, shipped map[string]bool) (string, error) { - return writeCompiledRelease(m, outputDirectory, stemcellSlug, blobsDirectory, initialTarball, sourcePackages, compiledPackages, CompileOptions{ExcludeCompileTimePackages: true}, shipped) + m.Packages = sourcePackages + return writeCompiledRelease(m, outputDirectory, stemcellSlug, blobsDirectory, initialTarball, compiledPackages, CompileOptions{ExcludeCompileTimePackages: true}, shipped) } diff --git a/releasetarball/fixtures_test.go b/releasetarball/fixtures_test.go new file mode 100644 index 000000000..fbb00c8c5 --- /dev/null +++ b/releasetarball/fixtures_test.go @@ -0,0 +1,414 @@ +//go:build fixtures + +package releasetarball_test + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "crypto/sha1" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "os" + "path/filepath" + "slices" + "strings" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/cloudfoundry/bosh-agent/v2/releasetarball" + "github.com/cloudfoundry/bosh-agent/v2/releasetarball/internal/fakes" + "github.com/cloudfoundry/bosh-agent/v2/settings/directories" + "github.com/cloudfoundry/bosh-cli/v7/release/manifest" + "gopkg.in/yaml.v3" +) + +var _ = Describe("Compile fixtures", func() { + entries := []TableEntry{ + Entry("source release", filepath.Join("testdata", "log-cache-release-3.0.9.tgz")), + Entry("compiled release", filepath.Join("testdata", "log-cache-3.0.9-banana-slug-1.23.tgz")), + } + if fixtureDir := os.Getenv("FIXTURES"); fixtureDir != "" { + fixtures, err := filepath.Glob(filepath.Join(fixtureDir, "*.tgz")) + if err != nil || len(fixtures) == 0 { + It("requires readable fixture tarballs", func() { + Expect(err).NotTo(HaveOccurred()) + Expect(fixtures).NotTo(BeEmpty(), "FIXTURES must contain at least one *.tgz file") + }) + } else { + for _, fixture := range fixtures { + entries = append(entries, Entry(filepath.Base(fixture), fixture)) + } + } + } + DescribeTable("preserves compiled blobs and trims only build dependencies", func(fixture string) { + input := prepareFixture(GinkgoT(), fixture) + inputManifest := mustReleaseManifest(input) + shipped := jobPackageNames(GinkgoT(), input) + allPackages := packageNames(inputManifest.Packages) + kept := shipped + dropped := difference(allPackages, shipped) + + workDir := GinkgoT().TempDir() + plainDir := filepath.Join(workDir, "plain") + flaggedDir := filepath.Join(workDir, "flagged") + blobs := directories.NewProvider(filepath.Join(workDir, "blobs")) + Expect(os.MkdirAll(blobs.BlobsDir(), 0o755)).To(Succeed()) + Expect(os.MkdirAll(plainDir, 0o755)).To(Succeed()) + Expect(os.MkdirAll(flaggedDir, 0o755)).To(Succeed()) + pkgCompiler := new(fakes.Compiler) + pkgCompiler.CompileCalls(fakeCompilation(blobs)) + + plainPath, plain := compileFixture(pkgCompiler, input, blobs.BlobsDir(), plainDir, false) + flaggedPath, flagged := compileFixture(pkgCompiler, input, blobs.BlobsDir(), flaggedDir, true) + Expect(packageNamesCompiled(flagged.CompiledPkgs)).To(Equal(kept)) + Expect(packageNamesCompileTime(flagged.CompileTimePackages)).To(Equal(dropped)) + assertCompiledBlobs(flaggedPath, flagged.CompiledPkgs) + assertCompiledBlobs(plainPath, plain.CompiledPkgs) + assertKeptPackages(inputManifest, flagged, kept) + + if len(dropped) == 0 { + plainBytes, err := os.ReadFile(plainPath) + Expect(err).NotTo(HaveOccurred()) + flaggedBytes, err := os.ReadFile(flaggedPath) + Expect(err).NotTo(HaveOccurred()) + Expect(sha256.Sum256(flaggedBytes)).To(Equal(sha256.Sum256(plainBytes))) + } else { + wantVersion := inputManifest.Version + "+trimmed" + if strings.Contains(inputManifest.Version, "+") { + wantVersion = inputManifest.Version + ".trimmed" + } + Expect(flagged.Version).To(Equal(wantVersion)) + } + + copyFixtureOutputs(fixture, flaggedPath, plainPath) + GinkgoWriter.Printf("fixture %s: flagged=%s plain=%s kept=%d dropped=%v\n", filepath.Base(fixture), flaggedPath, plainPath, len(kept), dropped) + }, entries) +}) + +func prepareFixture(t GinkgoTInterface, fixture string) string { + GinkgoHelper() + dir := t.TempDir() + extractFixture(t, fixture, dir) + manifestPath := filepath.Join(dir, "release.MF") + raw, err := os.ReadFile(manifestPath) + Expect(err).NotTo(HaveOccurred()) + var release manifest.Manifest + Expect(yaml.Unmarshal(raw, &release)).To(Succeed()) + if len(release.CompiledPkgs) > 0 && len(release.Packages) == 0 { + var doc map[string]any + Expect(yaml.Unmarshal(raw, &doc)).To(Succeed()) + for _, pkg := range release.CompiledPkgs { + release.Packages = append(release.Packages, manifest.PackageRef{ + Name: pkg.Name, Version: pkg.Version, Fingerprint: pkg.Fingerprint, SHA1: pkg.SHA1, Dependencies: slices.Clone(pkg.Dependencies), + }) + oldBlob := filepath.Join(dir, "compiled_packages", pkg.Name+".tgz") + newDir := filepath.Join(dir, "packages") + Expect(os.MkdirAll(newDir, 0o755)).To(Succeed()) + Expect(os.Rename(oldBlob, filepath.Join(newDir, pkg.Name+".tgz"))).To(Succeed()) + } + delete(doc, "compiled_packages") + packages := make([]any, 0, len(release.Packages)) + for _, pkg := range release.Packages { + packages = append(packages, map[string]any{ + "name": pkg.Name, "version": pkg.Version, "fingerprint": pkg.Fingerprint, + "sha1": pkg.SHA1, "dependencies": pkg.Dependencies, + }) + } + doc["packages"] = packages + converted, err := yaml.Marshal(doc) + Expect(err).NotTo(HaveOccurred()) + Expect(os.WriteFile(manifestPath, converted, 0o644)).To(Succeed()) + } + return createTarballFromDir(t, dir) +} + +func extractFixture(t GinkgoTInterface, fixture, dir string) { + GinkgoHelper() + f, err := os.Open(fixture) + Expect(err).NotTo(HaveOccurred()) + defer closeAndIgnoreError(f) + gz, err := gzip.NewReader(f) + Expect(err).NotTo(HaveOccurred()) + defer closeAndIgnoreError(gz) + tr := tar.NewReader(gz) + for { + h, err := tr.Next() + if err == io.EOF { + break + } + Expect(err).NotTo(HaveOccurred()) + clean := filepath.Clean(filepath.FromSlash(h.Name)) + Expect(filepath.IsLocal(clean)).To(BeTrue(), "invalid tar path %q", h.Name) + dst := filepath.Join(dir, clean) + switch h.Typeflag { + case tar.TypeDir: + Expect(os.MkdirAll(dst, 0o755)).To(Succeed()) + case tar.TypeReg: + Expect(os.MkdirAll(filepath.Dir(dst), 0o755)).To(Succeed()) + out, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644) + Expect(err).NotTo(HaveOccurred()) + _, copyErr := io.Copy(out, tr) + Expect(out.Close()).To(Succeed()) + Expect(copyErr).NotTo(HaveOccurred()) + default: + Fail(fmt.Sprintf("fixture contains unsupported tar entry %q", h.Name)) + } + } +} + +func createTarballFromDir(t GinkgoTInterface, dir string) string { + GinkgoHelper() + var directories, files []string + Expect(filepath.WalkDir(dir, func(p string, entry os.DirEntry, err error) error { + if err != nil { + return err + } + if p == dir { + return nil + } + rel, err := filepath.Rel(dir, p) + if err != nil { + return err + } + if entry.IsDir() { + directories = append(directories, filepath.ToSlash(rel)) + } else { + files = append(files, filepath.ToSlash(rel)) + } + return nil + })).To(Succeed()) + slices.Sort(directories) + slices.Sort(files) + output := filepath.Join(t.TempDir(), "input.tgz") + f, err := os.Create(output) + Expect(err).NotTo(HaveOccurred()) + gw := gzip.NewWriter(f) + tw := tar.NewWriter(gw) + for _, name := range directories { + Expect(tw.WriteHeader(&tar.Header{Name: name + "/", Mode: 0o755, Typeflag: tar.TypeDir})).To(Succeed()) + } + for _, name := range files { + data, err := os.ReadFile(filepath.Join(dir, filepath.FromSlash(name))) + Expect(err).NotTo(HaveOccurred()) + Expect(tw.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: int64(len(data)), Typeflag: tar.TypeReg})).To(Succeed()) + _, err = tw.Write(data) + Expect(err).NotTo(HaveOccurred()) + } + Expect(tw.Close()).To(Succeed()) + Expect(gw.Close()).To(Succeed()) + Expect(f.Close()).To(Succeed()) + return output +} + +func assertCompiledBlobs(tarball string, packages []manifest.CompiledPackageRef) { + GinkgoHelper() + f, err := os.Open(tarball) + Expect(err).NotTo(HaveOccurred()) + defer closeAndIgnoreError(f) + gz, err := gzip.NewReader(f) + Expect(err).NotTo(HaveOccurred()) + defer closeAndIgnoreError(gz) + tr := tar.NewReader(gz) + entries := map[string][][]byte{} + for { + h, err := tr.Next() + if err == io.EOF { + break + } + Expect(err).NotTo(HaveOccurred()) + contents, err := io.ReadAll(tr) + Expect(err).NotTo(HaveOccurred()) + name := strings.TrimPrefix(h.Name, "./") + entries[name] = append(entries[name], contents) + } + var names []string + for name := range entries { + if strings.HasPrefix(name, "compiled_packages/") && strings.HasSuffix(name, ".tgz") { + names = append(names, name) + } + } + expected := make([]string, 0, len(packages)) + for _, pkg := range packages { + expected = append(expected, "compiled_packages/"+pkg.Name+".tgz") + } + Expect(names).To(ConsistOf(expected), "archive must contain exactly the manifest's compiled blobs") + for _, pkg := range packages { + name := "compiled_packages/" + pkg.Name + ".tgz" + contents := entries[name] + Expect(contents).To(HaveLen(1), name) + if strings.HasPrefix(pkg.SHA1, "sha256:") { + actual := sha256.Sum256(contents[0]) + Expect("sha256:"+hex.EncodeToString(actual[:])).To(Equal(pkg.SHA1), name) + } else { + actual := sha1.Sum(contents[0]) + Expect(hex.EncodeToString(actual[:])).To(Equal(pkg.SHA1), name) + } + } +} + +func compileFixture(pkgCompiler *fakes.Compiler, input, blobsDir, outputDir string, exclude bool) (string, manifest.Manifest) { + GinkgoHelper() + output, err := releasetarball.Compile(pkgCompiler, input, blobsDir, outputDir, "banana-slug/1.23", releasetarball.CompileOptions{ExcludeCompileTimePackages: exclude}) + Expect(err).NotTo(HaveOccurred()) + return output, mustReleaseManifest(output) +} + +type jobManifest struct { + Packages []string `yaml:"packages"` +} + +func jobPackageNames(t GinkgoTInterface, input string) []string { + GinkgoHelper() + f, err := os.Open(input) + Expect(err).NotTo(HaveOccurred()) + defer closeAndIgnoreError(f) + gz, err := gzip.NewReader(f) + Expect(err).NotTo(HaveOccurred()) + defer closeAndIgnoreError(gz) + tr := tar.NewReader(gz) + var names []string + for { + h, err := tr.Next() + if err == io.EOF { + break + } + Expect(err).NotTo(HaveOccurred()) + if !strings.HasPrefix(h.Name, "jobs/") || !strings.HasSuffix(h.Name, ".tgz") { + continue + } + jobBytes, err := io.ReadAll(tr) + Expect(err).NotTo(HaveOccurred()) + jobGz, err := gzip.NewReader(bytes.NewReader(jobBytes)) + Expect(err).NotTo(HaveOccurred()) + jobTar := tar.NewReader(jobGz) + for { + jobHeader, err := jobTar.Next() + if err == io.EOF { + break + } + Expect(err).NotTo(HaveOccurred()) + if strings.TrimPrefix(jobHeader.Name, "./") != "job.MF" { + continue + } + var job jobManifest + Expect(yaml.NewDecoder(jobTar).Decode(&job)).To(Succeed()) + names = append(names, job.Packages...) + } + Expect(jobGz.Close()).To(Succeed()) + } + return uniqueSorted(names) +} + +func assertKeptPackages(source, output manifest.Manifest, kept []string) { + GinkgoHelper() + sourcePackages := map[string]manifest.PackageRef{} + for _, pkg := range source.Packages { + sourcePackages[pkg.Name] = pkg + } + keptSet := make(map[string]bool, len(kept)) + for _, name := range kept { + keptSet[name] = true + } + for _, compiled := range output.CompiledPkgs { + original := sourcePackages[compiled.Name] + var dependencies []string + for _, dependency := range original.Dependencies { + if keptSet[dependency] { + dependencies = append(dependencies, dependency) + } + } + Expect(slices.Equal(compiled.Dependencies, dependencies)).To(BeTrue(), compiled.Name) + if dependencyClosureContainsDropped(original.Name, sourcePackages, keptSet, map[string]bool{}) { + Expect(compiled.OriginalFingerprint).NotTo(BeEmpty(), compiled.Name) + Expect(compiled.OriginalFingerprint).NotTo(Equal(compiled.Fingerprint), compiled.Name) + Expect(compiled.Version).To(Equal(compiled.Fingerprint), compiled.Name) + } else { + Expect(compiled.OriginalFingerprint).To(BeEmpty(), compiled.Name) + Expect(compiled.Version).To(Equal(original.Version), compiled.Name) + Expect(compiled.Fingerprint).To(Equal(original.Fingerprint), compiled.Name) + } + } +} + +func dependencyClosureContainsDropped(name string, packages map[string]manifest.PackageRef, kept, visited map[string]bool) bool { + if visited[name] { + return false + } + visited[name] = true + for _, dependency := range packages[name].Dependencies { + if !kept[dependency] || dependencyClosureContainsDropped(dependency, packages, kept, visited) { + return true + } + } + return false +} + +func copyFixtureOutputs(fixture, flagged, plain string) { + GinkgoHelper() + outputDir := os.Getenv("FIXTURES_OUT") + if outputDir == "" { + return + } + Expect(os.MkdirAll(outputDir, 0o755)).To(Succeed()) + base := filepath.Base(fixture) + for _, output := range []struct{ source, suffix string }{{flagged, ".flagged.tgz"}, {plain, ".plain.tgz"}} { + data, err := os.ReadFile(output.source) + Expect(err).NotTo(HaveOccurred()) + Expect(os.WriteFile(filepath.Join(outputDir, strings.TrimSuffix(base, ".tgz")+output.suffix), data, 0o644)).To(Succeed()) + } +} + +func packageNames(packages []manifest.PackageRef) []string { + names := make([]string, 0, len(packages)) + for _, pkg := range packages { + names = append(names, pkg.Name) + } + return uniqueSorted(names) +} + +func packageNamesCompiled(packages []manifest.CompiledPackageRef) []string { + names := make([]string, 0, len(packages)) + for _, pkg := range packages { + names = append(names, pkg.Name) + } + return uniqueSorted(names) +} + +func packageNamesCompileTime(packages []manifest.CompileTimePackageRef) []string { + names := make([]string, 0, len(packages)) + for _, pkg := range packages { + names = append(names, pkg.Name) + } + return uniqueSorted(names) +} + +func uniqueSorted(names []string) []string { + set := map[string]bool{} + for _, name := range names { + set[name] = true + } + result := make([]string, 0, len(set)) + for name := range set { + result = append(result, name) + } + slices.Sort(result) + return result +} + +func difference(left, right []string) []string { + rightSet := map[string]bool{} + for _, name := range right { + rightSet[name] = true + } + var result []string + for _, name := range left { + if !rightSet[name] { + result = append(result, name) + } + } + return uniqueSorted(result) +} diff --git a/releasetarball/identity.go b/releasetarball/identity.go index e1ae1f2e4..e4020c7d9 100644 --- a/releasetarball/identity.go +++ b/releasetarball/identity.go @@ -4,7 +4,6 @@ import ( "bytes" "crypto/sha1" "crypto/sha256" - "encoding/hex" "encoding/json" "fmt" "slices" @@ -20,61 +19,32 @@ type rewrittenPkg struct { Rewritten bool } -func dependencyKey(name string, all []manifest.PackageRef) (string, error) { - packages := make(map[string]manifest.PackageRef, len(all)) - for _, p := range all { - packages[p.Name] = p - } - root, ok := packages[name] - if !ok { - return "", fmt.Errorf("unknown package %q", name) - } - - var arrayify func(manifest.PackageRef) ([]any, error) - arrayify = func(p manifest.PackageRef) ([]any, error) { - result := []any{p.Name, p.Version} - if len(p.Dependencies) > 0 { - dependencies := slices.Clone(p.Dependencies) - slices.Sort(dependencies) - children := make([]any, 0, len(dependencies)) - for _, dependency := range dependencies { - child, ok := packages[dependency] - if !ok { - return nil, fmt.Errorf("unknown dependency %q of package %q", dependency, p.Name) - } - childKey, err := arrayify(child) - if err != nil { - return nil, err - } - children = append(children, childKey) - } - result = append(result, children) - } - return result, nil - } - - dependencies := slices.Clone(root.Dependencies) +// dependencyTree follows the director's KeyGenerator#arrayify, including sorted +// children and omitted empty child arrays. Keep the director vectors in sync: +// https://github.com/cloudfoundry/bosh/blob/0410768ff620384100a277ff0683274970bc2c56/src/bosh-director/lib/bosh/director/key_generator.rb +// Compile checks for cycles before traversing this graph. +func dependencyTree(p manifest.PackageRef, packages map[string]manifest.PackageRef, shipped map[string]bool) ([]any, bool, error) { + dependencies := slices.Clone(p.Dependencies) slices.Sort(dependencies) - key := make([]any, 0, len(dependencies)) - for _, dependency := range dependencies { - p, ok := packages[dependency] + tree := make([]any, 0, len(dependencies)) + trimmed := false + for _, name := range dependencies { + child, ok := packages[name] if !ok { - return "", fmt.Errorf("unknown dependency %q of package %q", dependency, name) + return nil, false, fmt.Errorf("unknown dependency %q of package %q", name, p.Name) } - dependencyKey, err := arrayify(p) + children, childTrimmed, err := dependencyTree(child, packages, shipped) if err != nil { - return "", err + return nil, false, err } - key = append(key, dependencyKey) - } - - var output bytes.Buffer - encoder := json.NewEncoder(&output) - encoder.SetEscapeHTML(false) - if err := encoder.Encode(key); err != nil { - return "", err + entry := []any{child.Name, child.Version} + if len(children) > 0 { + entry = append(entry, children) + } + tree = append(tree, entry) + trimmed = trimmed || !shipped[name] || childTrimmed } - return strings.TrimSuffix(output.String(), "\n"), nil + return tree, trimmed, nil } func rewriteIdentities(all []manifest.PackageRef, shipped map[string]bool) (map[string]rewrittenPkg, error) { @@ -82,70 +52,50 @@ func rewriteIdentities(all []manifest.PackageRef, shipped map[string]bool) (map[ for _, p := range all { packages[p.Name] = p } - result := make(map[string]rewrittenPkg) for _, p := range all { if !shipped[p.Name] { continue } - key, err := dependencyKey(p.Name, all) + tree, trimmed, err := dependencyTree(p, packages, shipped) if err != nil { return nil, err } - - rewritten := false - var checkClosure func(manifest.PackageRef) error - checkClosure = func(current manifest.PackageRef) error { - for _, dependency := range current.Dependencies { - child, ok := packages[dependency] - if !ok { - return fmt.Errorf("unknown dependency %q of package %q", dependency, current.Name) - } - if !shipped[dependency] { - rewritten = true - } - if err := checkClosure(child); err != nil { - return err - } - } - return nil - } - if err := checkClosure(p); err != nil { - return nil, err - } - identity := rewrittenPkg{ Fingerprint: p.Fingerprint, OriginalFingerprint: p.Fingerprint, Dependencies: p.Dependencies, - Rewritten: rewritten, + Rewritten: trimmed, } - if rewritten { - var payload strings.Builder - payload.WriteString(p.Fingerprint) - payload.WriteByte('\n') - payload.WriteString(key) - payload.WriteByte('\n') - keptDependencies := make([]string, 0, len(p.Dependencies)) - for _, dependency := range p.Dependencies { - if shipped[dependency] { - keptDependencies = append(keptDependencies, dependency) + if trimmed { + identity.Dependencies = make([]string, 0, len(p.Dependencies)) + for _, name := range p.Dependencies { + if shipped[name] { + identity.Dependencies = append(identity.Dependencies, name) } } - slices.Sort(keptDependencies) - payload.WriteString(strings.Join(keptDependencies, ",")) - + slices.Sort(identity.Dependencies) + var key bytes.Buffer + encoder := json.NewEncoder(&key) + encoder.SetEscapeHTML(false) + if err := encoder.Encode(tree); err != nil { + return nil, err + } + // Preserve the existing trimmed identity format: source fingerprint, + // director dependency JSON, then comma-separated kept dependencies. + dependencyKey := strings.TrimSuffix(key.String(), "\n") + keptDependencies := strings.Join(identity.Dependencies, ",") + payload := p.Fingerprint + "\n" + dependencyKey + "\n" + keptDependencies switch len(p.Fingerprint) { case 40: - digest := sha1.Sum([]byte(payload.String())) - identity.Fingerprint = hex.EncodeToString(digest[:]) + digest := sha1.Sum([]byte(payload)) + identity.Fingerprint = fmt.Sprintf("%x", digest) case 64: - digest := sha256.Sum256([]byte(payload.String())) - identity.Fingerprint = hex.EncodeToString(digest[:]) + digest := sha256.Sum256([]byte(payload)) + identity.Fingerprint = fmt.Sprintf("%x", digest) default: return nil, fmt.Errorf("package %q has unsupported fingerprint length %d", p.Name, len(p.Fingerprint)) } - identity.Dependencies = keptDependencies } result[p.Name] = identity } diff --git a/releasetarball/identity_test.go b/releasetarball/identity_test.go index 65450d0db..282837fb5 100644 --- a/releasetarball/identity_test.go +++ b/releasetarball/identity_test.go @@ -336,29 +336,33 @@ func caseNamed(name string) identityCase { } var _ = Describe("package identities", func() { - for _, c := range identityCases { - It("computes identities for "+c.name, func() { - shipped := shippedSet(c.shipped) - result, err := releasetarball.RewriteIdentitiesForTest(c.packages, shipped) + DescribeTable("computes director-compatible identities", func(c identityCase) { + shipped := shippedSet(c.shipped) + result, err := releasetarball.RewriteIdentitiesForTest(c.packages, shipped) + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(HaveLen(len(c.expected))) + for name, expected := range c.expected { + actual, exists := result[name] + Expect(exists).To(BeTrue(), "package %q should be included", name) + key, err := releasetarball.DependencyKeyForTest(name, c.packages) Expect(err).NotTo(HaveOccurred()) - Expect(result).To(HaveLen(len(c.expected))) - for name, expected := range c.expected { - actual, exists := result[name] - Expect(exists).To(BeTrue(), "package %q should be included", name) - key, err := releasetarball.DependencyKeyForTest(name, c.packages) - Expect(err).NotTo(HaveOccurred()) - Expect(key).To(Equal(expected.dependencyKey), name) - Expect(actual.Rewritten).To(Equal(expected.rewritten), name) - Expect(actual.Fingerprint).To(Equal(expected.fingerprint), name) - Expect(actual.Dependencies).To(Equal(expected.dependencies), name) - } - for _, p := range c.packages { - if !shipped[p.Name] { - Expect(result).NotTo(HaveKey(p.Name), "dropped package %q should not be included", p.Name) - } + Expect(key).To(Equal(expected.dependencyKey), name) + Expect(actual.Rewritten).To(Equal(expected.rewritten), name) + Expect(actual.Fingerprint).To(Equal(expected.fingerprint), name) + Expect(actual.Dependencies).To(Equal(expected.dependencies), name) + } + for _, p := range c.packages { + if !shipped[p.Name] { + Expect(result).NotTo(HaveKey(p.Name), "dropped package %q should not be included", p.Name) } - }) - } + } + }, func() []TableEntry { + entries := make([]TableEntry, 0, len(identityCases)) + for _, c := range identityCases { + entries = append(entries, Entry(c.name, c)) + } + return entries + }()) It("keeps the fingerprint of a package that loses no dependencies", func() { c := caseNamed("leaf-nothing-dropped") diff --git a/vendor/github.com/onsi/gomega/gstruct/elements.go b/vendor/github.com/onsi/gomega/gstruct/elements.go new file mode 100644 index 000000000..4a6114ab8 --- /dev/null +++ b/vendor/github.com/onsi/gomega/gstruct/elements.go @@ -0,0 +1,235 @@ +// untested sections: 6 + +package gstruct + +import ( + "errors" + "fmt" + "reflect" + "runtime/debug" + "strconv" + + "github.com/onsi/gomega/format" + errorsutil "github.com/onsi/gomega/gstruct/errors" + "github.com/onsi/gomega/types" +) + +// MatchAllElements succeeds if every element of a slice matches the element matcher it maps to +// through the id function, and every element matcher is matched. +// +// idFn := func(element any) string { +// return fmt.Sprintf("%v", element) +// } +// +// Expect([]string{"a", "b"}).To(MatchAllElements(idFn, Elements{ +// "a": Equal("a"), +// "b": Equal("b"), +// })) +func MatchAllElements(identifier Identifier, elements Elements) types.GomegaMatcher { + return &ElementsMatcher{ + Identifier: identifier, + Elements: elements, + } +} + +// MatchAllElementsWithIndex succeeds if every element of a slice matches the element matcher it maps to +// through the id with index function, and every element matcher is matched. +// +// idFn := func(index int, element any) string { +// return strconv.Itoa(index) +// } +// +// Expect([]string{"a", "b"}).To(MatchAllElements(idFn, Elements{ +// "0": Equal("a"), +// "1": Equal("b"), +// })) +func MatchAllElementsWithIndex(identifier IdentifierWithIndex, elements Elements) types.GomegaMatcher { + return &ElementsMatcher{ + Identifier: identifier, + Elements: elements, + } +} + +// MatchElements succeeds if each element of a slice matches the element matcher it maps to +// through the id function. It can ignore extra elements and/or missing elements. +// +// idFn := func(element any) string { +// return fmt.Sprintf("%v", element) +// } +// +// Expect([]string{"a", "b", "c"}).To(MatchElements(idFn, IgnoreExtras, Elements{ +// "a": Equal("a"), +// "b": Equal("b"), +// })) +// Expect([]string{"a", "c"}).To(MatchElements(idFn, IgnoreMissing, Elements{ +// "a": Equal("a"), +// "b": Equal("b"), +// "c": Equal("c"), +// "d": Equal("d"), +// })) +func MatchElements(identifier Identifier, options Options, elements Elements) types.GomegaMatcher { + return &ElementsMatcher{ + Identifier: identifier, + Elements: elements, + IgnoreExtras: options&IgnoreExtras != 0, + IgnoreMissing: options&IgnoreMissing != 0, + AllowDuplicates: options&AllowDuplicates != 0, + } +} + +// MatchElementsWithIndex succeeds if each element of a slice matches the element matcher it maps to +// through the id with index function. It can ignore extra elements and/or missing elements. +// +// idFn := func(index int, element any) string { +// return strconv.Itoa(index) +// } +// +// Expect([]string{"a", "b", "c"}).To(MatchElements(idFn, IgnoreExtras, Elements{ +// "0": Equal("a"), +// "1": Equal("b"), +// })) +// Expect([]string{"a", "c"}).To(MatchElements(idFn, IgnoreMissing, Elements{ +// "0": Equal("a"), +// "1": Equal("b"), +// "2": Equal("c"), +// "3": Equal("d"), +// })) +func MatchElementsWithIndex(identifier IdentifierWithIndex, options Options, elements Elements) types.GomegaMatcher { + return &ElementsMatcher{ + Identifier: identifier, + Elements: elements, + IgnoreExtras: options&IgnoreExtras != 0, + IgnoreMissing: options&IgnoreMissing != 0, + AllowDuplicates: options&AllowDuplicates != 0, + } +} + +// ElementsMatcher is a NestingMatcher that applies custom matchers to each element of a slice mapped +// by the Identifier function. +// TODO: Extend this to work with arrays & maps (map the key) as well. +type ElementsMatcher struct { + // Matchers for each element. + Elements Elements + // Function mapping an element to the string key identifying its matcher. + Identifier Identify + + // Whether to ignore extra elements or consider it an error. + IgnoreExtras bool + // Whether to ignore missing elements or consider it an error. + IgnoreMissing bool + // Whether to key duplicates when matching IDs. + AllowDuplicates bool + + // State. + failures []error +} + +// Element ID to matcher. +type Elements map[string]types.GomegaMatcher + +// Function for identifying (mapping) elements. +type Identifier func(element any) string + +// Calls the underlying function with the provided params. +// Identifier drops the index. +func (i Identifier) WithIndexAndElement(index int, element any) string { + return i(element) +} + +// Uses the index and element to generate an element name +type IdentifierWithIndex func(index int, element any) string + +// Calls the underlying function with the provided params. +// IdentifierWithIndex uses the index. +func (i IdentifierWithIndex) WithIndexAndElement(index int, element any) string { + return i(index, element) +} + +// Interface for identifying the element +type Identify interface { + WithIndexAndElement(i int, element any) string +} + +// IndexIdentity is a helper function for using an index as +// the key in the element map +func IndexIdentity(index int, _ any) string { + return strconv.Itoa(index) +} + +func (m *ElementsMatcher) Match(actual any) (success bool, err error) { + if reflect.TypeOf(actual).Kind() != reflect.Slice { + return false, fmt.Errorf("%v is type %T, expected slice", actual, actual) + } + + m.failures = m.matchElements(actual) + if len(m.failures) > 0 { + return false, nil + } + return true, nil +} + +func (m *ElementsMatcher) matchElements(actual any) (errs []error) { + // Provide more useful error messages in the case of a panic. + defer func() { + if err := recover(); err != nil { + errs = append(errs, fmt.Errorf("panic checking %+v: %v\n%s", actual, err, debug.Stack())) + } + }() + + val := reflect.ValueOf(actual) + elements := map[string]bool{} + for i := 0; i < val.Len(); i++ { + element := val.Index(i).Interface() + id := m.Identifier.WithIndexAndElement(i, element) + if elements[id] { + if !m.AllowDuplicates { + errs = append(errs, fmt.Errorf("found duplicate element ID %s", id)) + continue + } + } + elements[id] = true + + matcher, expected := m.Elements[id] + if !expected { + if !m.IgnoreExtras { + errs = append(errs, fmt.Errorf("unexpected element %s", id)) + } + continue + } + + match, err := matcher.Match(element) + if match { + continue + } + + if err == nil { + if nesting, ok := matcher.(errorsutil.NestingMatcher); ok { + err = errorsutil.AggregateError(nesting.Failures()) + } else { + err = errors.New(matcher.FailureMessage(element)) + } + } + errs = append(errs, errorsutil.Nest(fmt.Sprintf("[%s]", id), err)) + } + + for id := range m.Elements { + if !elements[id] && !m.IgnoreMissing { + errs = append(errs, fmt.Errorf("missing expected element %s", id)) + } + } + + return errs +} + +func (m *ElementsMatcher) FailureMessage(actual any) (message string) { + failure := errorsutil.AggregateError(m.failures) + return format.Message(actual, fmt.Sprintf("to match elements: %v", failure)) +} + +func (m *ElementsMatcher) NegatedFailureMessage(actual any) (message string) { + return format.Message(actual, "not to match elements") +} + +func (m *ElementsMatcher) Failures() []error { + return m.failures +} diff --git a/vendor/github.com/onsi/gomega/gstruct/errors/nested_types.go b/vendor/github.com/onsi/gomega/gstruct/errors/nested_types.go new file mode 100644 index 000000000..188492b21 --- /dev/null +++ b/vendor/github.com/onsi/gomega/gstruct/errors/nested_types.go @@ -0,0 +1,72 @@ +package errors + +import ( + "fmt" + "strings" + + "github.com/onsi/gomega/types" +) + +// A stateful matcher that nests other matchers within it and preserves the error types of the +// nested matcher failures. +type NestingMatcher interface { + types.GomegaMatcher + + // Returns the failures of nested matchers. + Failures() []error +} + +// An error type for labeling errors on deeply nested matchers. +type NestedError struct { + Path string + Err error +} + +func (e *NestedError) Error() string { + // Indent Errors. + indented := strings.Replace(e.Err.Error(), "\n", "\n\t", -1) + return fmt.Sprintf("%s:\n\t%v", e.Path, indented) +} + +// Create a NestedError with the given path. +// If err is a NestedError, prepend the path to it. +// If err is an AggregateError, recursively Nest each error. +func Nest(path string, err error) error { + if ag, ok := err.(AggregateError); ok { + var errs AggregateError + for _, e := range ag { + errs = append(errs, Nest(path, e)) + } + return errs + } + if ne, ok := err.(*NestedError); ok { + return &NestedError{ + Path: path + ne.Path, + Err: ne.Err, + } + } + return &NestedError{ + Path: path, + Err: err, + } +} + +// An error type for treating multiple errors as a single error. +type AggregateError []error + +// Error is part of the error interface. +func (err AggregateError) Error() string { + if len(err) == 0 { + // This should never happen, really. + return "" + } + if len(err) == 1 { + return err[0].Error() + } + result := fmt.Sprintf("[%s", err[0].Error()) + for i := 1; i < len(err); i++ { + result += fmt.Sprintf(", %s", err[i].Error()) + } + result += "]" + return result +} diff --git a/vendor/github.com/onsi/gomega/gstruct/fields.go b/vendor/github.com/onsi/gomega/gstruct/fields.go new file mode 100644 index 000000000..2f4685fc4 --- /dev/null +++ b/vendor/github.com/onsi/gomega/gstruct/fields.go @@ -0,0 +1,187 @@ +// untested sections: 6 + +package gstruct + +import ( + "errors" + "fmt" + "reflect" + "runtime/debug" + "strings" + "unicode" + + "github.com/onsi/gomega/format" + errorsutil "github.com/onsi/gomega/gstruct/errors" + "github.com/onsi/gomega/types" +) + +// MatchAllFields succeeds if every field of a struct matches the field matcher associated with +// it, and every element matcher is matched. +// +// actual := struct{ +// A int +// B []bool +// C string +// }{ +// A: 5, +// B: []bool{true, false}, +// C: "foo", +// } +// +// Expect(actual).To(MatchAllFields(Fields{ +// "A": Equal(5), +// "B": ConsistOf(true, false), +// "C": Equal("foo"), +// })) +func MatchAllFields(fields Fields) types.GomegaMatcher { + return &FieldsMatcher{ + Fields: fields, + } +} + +// MatchFields succeeds if each element of a struct matches the field matcher associated with +// it. It can ignore extra fields and/or missing fields. +// +// actual := struct{ +// A int +// B []bool +// C string +// }{ +// A: 5, +// B: []bool{true, false}, +// C: "foo", +// } +// +// Expect(actual).To(MatchFields(IgnoreExtras, Fields{ +// "A": Equal(5), +// "B": ConsistOf(true, false), +// })) +// Expect(actual).To(MatchFields(IgnoreMissing, Fields{ +// "A": Equal(5), +// "B": ConsistOf(true, false), +// "C": Equal("foo"), +// "D": Equal("extra"), +// })) +func MatchFields(options Options, fields Fields) types.GomegaMatcher { + return &FieldsMatcher{ + Fields: fields, + IgnoreExtras: options&IgnoreExtras != 0, + IgnoreUnexportedExtras: options&IgnoreUnexportedExtras != 0, + IgnoreMissing: options&IgnoreMissing != 0, + } +} + +type FieldsMatcher struct { + // Matchers for each field. + Fields Fields + + // Whether to ignore extra elements or consider it an error. + IgnoreExtras bool + // Whether to ignore unexported extra elements or consider it an error. + IgnoreUnexportedExtras bool + // Whether to ignore missing elements or consider it an error. + IgnoreMissing bool + + // State. + failures []error +} + +// Field name to matcher. +type Fields map[string]types.GomegaMatcher + +func (m *FieldsMatcher) Match(actual any) (success bool, err error) { + if reflect.TypeOf(actual).Kind() != reflect.Struct { + return false, fmt.Errorf("%v is type %T, expected struct", actual, actual) + } + + m.failures = m.matchFields(actual) + if len(m.failures) > 0 { + return false, nil + } + return true, nil +} + +func isExported(fieldName string) bool { + if fieldName == "" { + return false + } + r := []rune(fieldName)[0] + return unicode.IsUpper(r) +} + +func (m *FieldsMatcher) matchFields(actual any) (errs []error) { + val := reflect.ValueOf(actual) + typ := val.Type() + fields := map[string]bool{} + for i := 0; i < val.NumField(); i++ { + fieldName := typ.Field(i).Name + fields[fieldName] = true + + err := func() (err error) { + // This test relies heavily on reflect, which tends to panic. + // Recover here to provide more useful error messages in that case. + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("panic checking %+v: %v\n%s", actual, r, debug.Stack()) + } + }() + + matcher, expected := m.Fields[fieldName] + if !expected { + if m.IgnoreUnexportedExtras && !isExported(fieldName) { + return nil + } + if !m.IgnoreExtras { + return fmt.Errorf("unexpected field %s: %+v", fieldName, actual) + } + return nil + } + + var field any + if _, isIgnoreMatcher := matcher.(*IgnoreMatcher) ; isIgnoreMatcher { + field = struct {}{} // the matcher does not care about the actual value + } else { + field = val.Field(i).Interface() + } + + match, err := matcher.Match(field) + if err != nil { + return err + } else if !match { + if nesting, ok := matcher.(errorsutil.NestingMatcher); ok { + return errorsutil.AggregateError(nesting.Failures()) + } + return errors.New(matcher.FailureMessage(field)) + } + return nil + }() + if err != nil { + errs = append(errs, errorsutil.Nest("."+fieldName, err)) + } + } + + for field := range m.Fields { + if !fields[field] && !m.IgnoreMissing { + errs = append(errs, fmt.Errorf("missing expected field %s", field)) + } + } + + return errs +} + +func (m *FieldsMatcher) FailureMessage(actual any) (message string) { + failures := make([]string, len(m.failures)) + for i := range m.failures { + failures[i] = m.failures[i].Error() + } + return format.Message(reflect.TypeOf(actual).Name(), + fmt.Sprintf("to match fields: {\n%v\n}\n", strings.Join(failures, "\n"))) +} + +func (m *FieldsMatcher) NegatedFailureMessage(actual any) (message string) { + return format.Message(actual, "not to match fields") +} + +func (m *FieldsMatcher) Failures() []error { + return m.failures +} diff --git a/vendor/github.com/onsi/gomega/gstruct/ignore.go b/vendor/github.com/onsi/gomega/gstruct/ignore.go new file mode 100644 index 000000000..c8238d58f --- /dev/null +++ b/vendor/github.com/onsi/gomega/gstruct/ignore.go @@ -0,0 +1,41 @@ +// untested sections: 2 + +package gstruct + +import ( + "github.com/onsi/gomega/types" +) + +// Ignore ignores the actual value and always succeeds. +// +// Expect(nil).To(Ignore()) +// Expect(true).To(Ignore()) +func Ignore() types.GomegaMatcher { + return &IgnoreMatcher{true} +} + +// Reject ignores the actual value and always fails. It can be used in conjunction with IgnoreMissing +// to catch problematic elements, or to verify tests are running. +// +// Expect(nil).NotTo(Reject()) +// Expect(true).NotTo(Reject()) +func Reject() types.GomegaMatcher { + return &IgnoreMatcher{false} +} + +// A matcher that either always succeeds or always fails. +type IgnoreMatcher struct { + Succeed bool +} + +func (m *IgnoreMatcher) Match(actual any) (bool, error) { + return m.Succeed, nil +} + +func (m *IgnoreMatcher) FailureMessage(_ any) (message string) { + return "Unconditional failure" +} + +func (m *IgnoreMatcher) NegatedFailureMessage(_ any) (message string) { + return "Unconditional success" +} diff --git a/vendor/github.com/onsi/gomega/gstruct/keys.go b/vendor/github.com/onsi/gomega/gstruct/keys.go new file mode 100644 index 000000000..807a450b6 --- /dev/null +++ b/vendor/github.com/onsi/gomega/gstruct/keys.go @@ -0,0 +1,126 @@ +// untested sections: 6 + +package gstruct + +import ( + "errors" + "fmt" + "reflect" + "runtime/debug" + "strings" + + "github.com/onsi/gomega/format" + errorsutil "github.com/onsi/gomega/gstruct/errors" + "github.com/onsi/gomega/types" +) + +func MatchAllKeys(keys Keys) types.GomegaMatcher { + return &KeysMatcher{ + Keys: keys, + } +} + +func MatchKeys(options Options, keys Keys) types.GomegaMatcher { + return &KeysMatcher{ + Keys: keys, + IgnoreExtras: options&IgnoreExtras != 0, + IgnoreMissing: options&IgnoreMissing != 0, + } +} + +type KeysMatcher struct { + // Matchers for each key. + Keys Keys + + // Whether to ignore extra keys or consider it an error. + IgnoreExtras bool + // Whether to ignore missing keys or consider it an error. + IgnoreMissing bool + + // State. + failures []error +} + +type Keys map[any]types.GomegaMatcher + +func (m *KeysMatcher) Match(actual any) (success bool, err error) { + if reflect.TypeOf(actual).Kind() != reflect.Map { + return false, fmt.Errorf("%v is type %T, expected map", actual, actual) + } + + m.failures = m.matchKeys(actual) + if len(m.failures) > 0 { + return false, nil + } + return true, nil +} + +func (m *KeysMatcher) matchKeys(actual any) (errs []error) { + actualValue := reflect.ValueOf(actual) + keys := map[any]bool{} + for _, keyValue := range actualValue.MapKeys() { + key := keyValue.Interface() + keys[key] = true + + err := func() (err error) { + // This test relies heavily on reflect, which tends to panic. + // Recover here to provide more useful error messages in that case. + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("panic checking %+v: %v\n%s", actual, r, debug.Stack()) + } + }() + + matcher, ok := m.Keys[key] + if !ok { + if !m.IgnoreExtras { + return fmt.Errorf("unexpected key %s: %+v", key, actual) + } + return nil + } + + valInterface := actualValue.MapIndex(keyValue).Interface() + + match, err := matcher.Match(valInterface) + if err != nil { + return err + } + + if !match { + if nesting, ok := matcher.(errorsutil.NestingMatcher); ok { + return errorsutil.AggregateError(nesting.Failures()) + } + return errors.New(matcher.FailureMessage(valInterface)) + } + return nil + }() + if err != nil { + errs = append(errs, errorsutil.Nest(fmt.Sprintf(".%#v", key), err)) + } + } + + for key := range m.Keys { + if !keys[key] && !m.IgnoreMissing { + errs = append(errs, fmt.Errorf("missing expected key %s", key)) + } + } + + return errs +} + +func (m *KeysMatcher) FailureMessage(actual any) (message string) { + failures := make([]string, len(m.failures)) + for i := range m.failures { + failures[i] = m.failures[i].Error() + } + return format.Message(reflect.TypeOf(actual).Name(), + fmt.Sprintf("to match keys: {\n%v\n}\n", strings.Join(failures, "\n"))) +} + +func (m *KeysMatcher) NegatedFailureMessage(actual any) (message string) { + return format.Message(actual, "not to match keys") +} + +func (m *KeysMatcher) Failures() []error { + return m.failures +} diff --git a/vendor/github.com/onsi/gomega/gstruct/pointer.go b/vendor/github.com/onsi/gomega/gstruct/pointer.go new file mode 100644 index 000000000..54c9557fe --- /dev/null +++ b/vendor/github.com/onsi/gomega/gstruct/pointer.go @@ -0,0 +1,59 @@ +// untested sections: 3 + +package gstruct + +import ( + "fmt" + "reflect" + + "github.com/onsi/gomega/format" + "github.com/onsi/gomega/types" +) + +// PointTo applies the given matcher to the value pointed to by actual. It fails if the pointer is +// nil. +// +// actual := 5 +// Expect(&actual).To(PointTo(Equal(5))) +func PointTo(matcher types.GomegaMatcher) types.GomegaMatcher { + return &PointerMatcher{ + Matcher: matcher, + } +} + +type PointerMatcher struct { + Matcher types.GomegaMatcher + + // Failure message. + failure string +} + +func (m *PointerMatcher) Match(actual any) (bool, error) { + val := reflect.ValueOf(actual) + + // return error if actual type is not a pointer + if val.Kind() != reflect.Ptr { + return false, fmt.Errorf("PointerMatcher expects a pointer but we have '%s'", val.Kind()) + } + + if !val.IsValid() || val.IsNil() { + m.failure = format.Message(actual, "not to be ") + return false, nil + } + + // Forward the value. + elem := val.Elem().Interface() + match, err := m.Matcher.Match(elem) + if !match { + m.failure = m.Matcher.FailureMessage(elem) + } + return match, err +} + +func (m *PointerMatcher) FailureMessage(_ any) (message string) { + return m.failure +} + +func (m *PointerMatcher) NegatedFailureMessage(actual any) (message string) { + return m.Matcher.NegatedFailureMessage(actual) +} diff --git a/vendor/github.com/onsi/gomega/gstruct/types.go b/vendor/github.com/onsi/gomega/gstruct/types.go new file mode 100644 index 000000000..a5f6c390b --- /dev/null +++ b/vendor/github.com/onsi/gomega/gstruct/types.go @@ -0,0 +1,19 @@ +package gstruct + +// Options is the type for options passed to some matchers. +type Options int + +const ( + //IgnoreExtras tells the matcher to ignore extra elements or fields, rather than triggering a failure. + IgnoreExtras Options = 1 << iota + //IgnoreMissing tells the matcher to ignore missing elements or fields, rather than triggering a failure. + IgnoreMissing + //AllowDuplicates tells the matcher to permit multiple members of the slice to produce the same ID when + //considered by the identifier function. All members that map to a given key must still match successfully + //with the matcher that is provided for that key. + AllowDuplicates + //IgnoreUnexportedExtras tells the matcher to ignore extra unexported fields, rather than triggering a failure. + //it is not possible to check the value of unexported fields, so this option is only useful when you want to + //check every exported fields, but you don't care about extra unexported fields. + IgnoreUnexportedExtras +) diff --git a/vendor/modules.txt b/vendor/modules.txt index 4ac8061f7..e0973c951 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -303,6 +303,8 @@ github.com/onsi/gomega/format github.com/onsi/gomega/gbytes github.com/onsi/gomega/gexec github.com/onsi/gomega/ghttp +github.com/onsi/gomega/gstruct +github.com/onsi/gomega/gstruct/errors github.com/onsi/gomega/internal github.com/onsi/gomega/internal/gutil github.com/onsi/gomega/matchers