feat: Add ParseMetadata() function to parity BHCE ingest - BED-7790

.gitignore

@@ -0,0 +1 @@
+fixtures/

README.md

@@ -22,18 +22,34 @@ Or you can clone the repo and run the following command from the top level:
 go install .
 ```
 
+# Benchmarking
+
+Use `chowbench` when you want to measure validation performance across one or more files without changing the normal `chow` CLI.
+
+```bash
+go run ./cmd/chowbench -runs 5 -warmup 1 payload-one.json payload-two.json
+```
+
+The harness loads the JSON schemas once, validates each file for the requested number of runs, and prints a table with byte size, status, average duration, min/max duration, and error counts.
+
+By default, invalid payloads are still measured and reported. Add `-strict` if invalid payloads should make the command exit non-zero:
+
+```bash
+go run ./cmd/chowbench -runs 5 -strict payload-one.json payload-two.json
+```
+
 # JSON Schema
 
 Want to add the OpenGraph schema to your JSON document?
 
 ```json
 {
-  "$schema": "https://raw.githubusercontent.com/SpecterOps/chow/refs/heads/main/pkg/validator/jsonschema/payload-schema.json"
+  "$schema": "https://raw.githubusercontent.com/SpecterOps/chow/refs/heads/main/pkg/payload/jsonschema/schema.json"
 }
 ```
 
 Most editors will ask you to trust the schema's source. Be sure to add the following URL to your trusted domains
 
 ```text
-https://raw.githubusercontent.com/SpecterOps/chow/refs/heads/main/pkg/validator/jsonschema/
+https://raw.githubusercontent.com/SpecterOps/chow/refs/heads/main/pkg/payload/jsonschema/
 ```

cmd/chowbench/main.go

@@ -0,0 +1,208 @@
+package main
+
+import (
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"text/tabwriter"
+	"time"
+
+	"github.com/specterops/chow/pkg/payload"
+)
+
+type durationSummary struct {
+	Avg time.Duration
+	Min time.Duration
+	Max time.Duration
+}
+
+type benchmarkResult struct {
+	File             string
+	Bytes            int64
+	Runs             int
+	Status           string
+	Error            string
+	CriticalErrors   int
+	ValidationErrors int
+	Durations        durationSummary
+}
+
+func main() {
+	var (
+		runs   int
+		warmup int
+		strict bool
+	)
+
+	flag.IntVar(&runs, "runs", 3, "number of measured validation runs per file")
+	flag.IntVar(&warmup, "warmup", 1, "number of unmeasured warmup validation runs per file")
+	flag.BoolVar(&strict, "strict", false, "exit non-zero when a file fails validation")
+	flag.Parse()
+
+	if err := run(os.Stdout, flag.Args(), runs, warmup, strict); err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+}
+
+func run(w io.Writer, files []string, runs int, warmup int, strict bool) error {
+	if runs < 1 {
+		return fmt.Errorf("-runs must be greater than 0")
+	}
+	if warmup < 0 {
+		return fmt.Errorf("-warmup must be 0 or greater")
+	}
+	if len(files) == 0 {
+		return fmt.Errorf("usage: chowbench [-runs N] [-warmup N] [-strict] file [file...]")
+	}
+
+	schema, err := payload.LoadSchema()
+	if err != nil {
+		return fmt.Errorf("load schema: %w", err)
+	}
+
+	results := make([]benchmarkResult, 0, len(files))
+	for _, file := range files {
+		result := benchmarkFile(file, schema, runs, warmup)
+		results = append(results, result)
+	}
+
+	if err := writeResults(w, results); err != nil {
+		return err
+	}
+	return exitErrorForResults(results, strict)
+}
+
+func benchmarkFile(file string, schema payload.Schema, runs int, warmup int) benchmarkResult {
+	result := benchmarkResult{
+		File: file,
+		Runs: runs,
+	}
+
+	if stat, err := os.Stat(file); err != nil {
+		result.Status = "error"
+		result.Error = err.Error()
+		return result
+	} else {
+		result.Bytes = stat.Size()
+	}
+
+	for i := 0; i < warmup; i++ {
+		_, _ = validateFile(file, schema)
+	}
+
+	durations := make([]time.Duration, 0, runs)
+	for i := 0; i < runs; i++ {
+		start := time.Now()
+		report, err := validateFile(file, schema)
+		durations = append(durations, time.Since(start))
+
+		result.Status, result.Error = statusForValidationResult(report, err)
+		result.CriticalErrors = len(report.CriticalErrors)
+		result.ValidationErrors = len(report.ValidationErrors)
+	}
+
+	result.Durations = summarizeDurations(durations)
+	return result
+}
+
+func validateFile(file string, schema payload.Schema) (payload.ValidationReport, error) {
+	reader, err := os.Open(file)
+	if err != nil {
+		return payload.ValidationReport{}, err
+	}
+	defer reader.Close()
+
+	validator := payload.NewValidator(reader, schema)
+	_, report, err := validator.ParseAndValidate()
+	return report, err
+}
+
+func summarizeDurations(durations []time.Duration) durationSummary {
+	if len(durations) == 0 {
+		return durationSummary{}
+	}
+
+	var total time.Duration
+	summary := durationSummary{
+		Min: durations[0],
+		Max: durations[0],
+	}
+
+	for _, duration := range durations {
+		total += duration
+		if duration < summary.Min {
+			summary.Min = duration
+		}
+		if duration > summary.Max {
+			summary.Max = duration
+		}
+	}
+
+	summary.Avg = total / time.Duration(len(durations))
+	return summary
+}
+
+func statusForValidationResult(report payload.ValidationReport, err error) (string, string) {
+	if err == nil {
+		return "ok", ""
+	}
+
+	if len(report.CriticalErrors) > 0 {
+		return "critical_error", err.Error()
+	}
+
+	if len(report.ValidationErrors) > 0 ||
+		errors.Is(err, payload.ErrValidationErrors) ||
+		errors.Is(err, payload.ErrMaxValidationErrors) {
+		return "validation_error", err.Error()
+	}
+
+	return "error", err.Error()
+}
+
+func exitErrorForResults(results []benchmarkResult, strict bool) error {
+	var hasValidationFailure bool
+	for _, result := range results {
+		switch result.Status {
+		case "error":
+			return fmt.Errorf("one or more files could not be benchmarked")
+		case "validation_error", "critical_error":
+			hasValidationFailure = true
+		}
+	}
+
+	if strict && hasValidationFailure {
+		return fmt.Errorf("one or more files failed validation")
+	}
+
+	return nil
+}
+
+func writeResults(w io.Writer, results []benchmarkResult) error {
+	tw := tabwriter.NewWriter(w, 0, 0, 2, ' ', 0)
+	if _, err := fmt.Fprintln(tw, "file\tbytes\truns\tstatus\tavg\tmin\tmax\tcritical\tvalidation\terror"); err != nil {
+		return err
+	}
+	for _, result := range results {
+		if _, err := fmt.Fprintf(
+			tw,
+			"%s\t%d\t%d\t%s\t%s\t%s\t%s\t%d\t%d\t%s\n",
+			result.File,
+			result.Bytes,
+			result.Runs,
+			result.Status,
+			result.Durations.Avg,
+			result.Durations.Min,
+			result.Durations.Max,
+			result.CriticalErrors,
+			result.ValidationErrors,
+			result.Error,
+		); err != nil {
+			return err
+		}
+	}
+	return tw.Flush()
+}

cmd/chowbench/main_test.go

@@ -0,0 +1,148 @@
+package main
+
+import (
+	"errors"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/specterops/chow/pkg/payload"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+var errWriteFailed = errors.New("write failed")
+
+type failingWriter struct{}
+
+func (failingWriter) Write([]byte) (int, error) {
+	return 0, errWriteFailed
+}
+
+func TestSummarizeDurations(t *testing.T) {
+	summary := summarizeDurations([]time.Duration{
+		3 * time.Millisecond,
+		1 * time.Millisecond,
+		2 * time.Millisecond,
+	})
+
+	assert.Equal(t, 2*time.Millisecond, summary.Avg)
+	assert.Equal(t, time.Millisecond, summary.Min)
+	assert.Equal(t, 3*time.Millisecond, summary.Max)
+}
+
+func TestRunReturnsWriteError(t *testing.T) {
+	file := filepath.Join(t.TempDir(), "payload.json")
+	require.NoError(t, os.WriteFile(file, []byte(`{"graph":{"nodes":[]}}`), 0o600))
+
+	err := run(failingWriter{}, []string{file}, 1, 0, false)
+
+	assert.ErrorIs(t, err, errWriteFailed)
+}
+
+func TestSummarizeDurationsEmpty(t *testing.T) {
+	assert.Equal(t, durationSummary{}, summarizeDurations(nil))
+}
+
+func TestStatusForValidationResult(t *testing.T) {
+	assertions := []struct {
+		name           string
+		report         payload.ValidationReport
+		err            error
+		expectedStatus string
+		expectedErr    string
+	}{
+		{
+			name:           "valid file",
+			expectedStatus: "ok",
+		},
+		{
+			name: "validation errors",
+			report: payload.ValidationReport{
+				ValidationErrors: []payload.ValidationError{{Location: "/graph/nodes[0]"}},
+			},
+			err:            payload.ErrValidationErrors,
+			expectedStatus: "validation_error",
+			expectedErr:    payload.ErrValidationErrors.Error(),
+		},
+		{
+			name: "critical errors",
+			report: payload.ValidationReport{
+				CriticalErrors: []payload.CriticalError{{Message: "bad file"}},
+			},
+			err:            payload.ErrInvalidFileConfiguration,
+			expectedStatus: "critical_error",
+			expectedErr:    payload.ErrInvalidFileConfiguration.Error(),
+		},
+		{
+			name:           "harness error",
+			err:            errors.New("open file: permission denied"),
+			expectedStatus: "error",
+			expectedErr:    "open file: permission denied",
+		},
+	}
+
+	for _, assertion := range assertions {
+		t.Run(assertion.name, func(t *testing.T) {
+			status, errText := statusForValidationResult(assertion.report, assertion.err)
+
+			assert.Equal(t, assertion.expectedStatus, status)
+			assert.Equal(t, assertion.expectedErr, errText)
+		})
+	}
+}
+
+func TestExitErrorForResults(t *testing.T) {
+	assertions := []struct {
+		name        string
+		results     []benchmarkResult
+		strict      bool
+		expectError bool
+	}{
+		{
+			name: "valid files",
+			results: []benchmarkResult{
+				{Status: "ok"},
+			},
+		},
+		{
+			name: "validation errors are allowed by default",
+			results: []benchmarkResult{
+				{Status: "validation_error"},
+			},
+		},
+		{
+			name: "validation errors fail in strict mode",
+			results: []benchmarkResult{
+				{Status: "validation_error"},
+			},
+			strict:      true,
+			expectError: true,
+		},
+		{
+			name: "harness errors always fail",
+			results: []benchmarkResult{
+				{Status: "error"},
+			},
+			expectError: true,
+		},
+	}
+
+	for _, assertion := range assertions {
+		t.Run(assertion.name, func(t *testing.T) {
+			err := exitErrorForResults(assertion.results, assertion.strict)
+			if assertion.expectError {
+				assert.Error(t, err)
+			} else {
+				assert.NoError(t, err)
+			}
+		})
+	}
+}
+
+func TestWriteResultsReturnsWriteError(t *testing.T) {
+	err := writeResults(failingWriter{}, []benchmarkResult{{File: "payload.json", Status: "ok"}})
+
+	assert.ErrorIs(t, err, errWriteFailed)
+}