From 6328fc7e51a5eb264832dd17ffe4d9bc27a2a3b4 Mon Sep 17 00:00:00 2001 From: betterclever Date: Mon, 9 Feb 2026 14:19:20 +0400 Subject: [PATCH 001/690] feat(runtime): add GCP config envs (dev/environment) --- .github/workflows/release.yml | 340 ---- .gitignore | 20 +- docker/PRODUCTION.md | 223 --- docker/README.md | 169 -- docker/SECURE-DEV-MODE.md | 126 -- docker/certs/.gitignore | 7 - docker/docker-compose.dev-ports.yml | 54 - docker/docker-compose.dev-secure.yml | 75 - docker/docker-compose.full.yml | 415 ----- docker/docker-compose.infra.yml | 242 --- docker/docker-compose.prod.yml | 97 -- docker/docker-compose.yml | 42 - .../init-db/01-create-instance-databases.sh | 39 - docker/loki/loki-config.yaml | 51 - docker/mcp-aws-cloudtrail/Dockerfile | 11 - docker/mcp-aws-cloudtrail/README.md | 22 - docker/mcp-aws-cloudwatch/Dockerfile | 14 - docker/mcp-aws-cloudwatch/README.md | 22 - docker/mcp-aws-suite/Dockerfile | 64 - docker/mcp-aws-suite/README.md | 66 - docker/mcp-aws-suite/named-servers.json | 16 - docker/mcp-stdio-proxy/Dockerfile | 14 - docker/mcp-stdio-proxy/README.md | 31 - docker/mcp-stdio-proxy/named-servers.json | 3 - docker/mcp-stdio-proxy/package.json | 14 - docker/mcp-stdio-proxy/server.mjs | 298 ---- docker/nginx/nginx.dev.conf | 242 --- docker/nginx/nginx.prod.conf | 208 --- docker/opensearch-dashboards.Dockerfile | 18 - docker/opensearch-dashboards.prod.yml | 66 - docker/opensearch-dashboards.yml | 33 - docker/opensearch-init.sh | 102 -- docker/opensearch-security/action_groups.yml | 61 - docker/opensearch-security/allowlist.yml | 13 - docker/opensearch-security/audit.yml | 30 - docker/opensearch-security/config.yml | 47 - .../docker-entrypoint-security.sh | 108 -- docker/opensearch-security/internal_users.yml | 72 - docker/opensearch-security/nodes_dn.yml | 12 - docker/opensearch-security/roles.yml | 177 -- docker/opensearch-security/roles_mapping.yml | 73 - docker/opensearch-security/tenants.yml | 28 - docker/opensearch-security/whitelist.yml | 13 - docker/opensearch.dev-secure.yml | 35 - docker/redpanda-console-config.yaml | 11 - docker/scripts/generate-certs.sh | 91 - docker/scripts/hash-password.sh | 37 - docker/scripts/security-init.sh | 157 -- install.sh | 1528 ----------------- scripts/db-reset-instance.sh | 77 - scripts/instance-clean.sh | 58 - 51 files changed, 16 insertions(+), 5756 deletions(-) delete mode 100644 .github/workflows/release.yml delete mode 100644 docker/PRODUCTION.md delete mode 100644 docker/README.md delete mode 100644 docker/SECURE-DEV-MODE.md delete mode 100644 docker/certs/.gitignore delete mode 100644 docker/docker-compose.dev-ports.yml delete mode 100644 docker/docker-compose.dev-secure.yml delete mode 100644 docker/docker-compose.full.yml delete mode 100644 docker/docker-compose.infra.yml delete mode 100644 docker/docker-compose.prod.yml delete mode 100644 docker/docker-compose.yml delete mode 100755 docker/init-db/01-create-instance-databases.sh delete mode 100644 docker/loki/loki-config.yaml delete mode 100644 docker/mcp-aws-cloudtrail/Dockerfile delete mode 100644 docker/mcp-aws-cloudtrail/README.md delete mode 100644 docker/mcp-aws-cloudwatch/Dockerfile delete mode 100644 docker/mcp-aws-cloudwatch/README.md delete mode 100644 docker/mcp-aws-suite/Dockerfile delete mode 100644 docker/mcp-aws-suite/README.md delete mode 100644 docker/mcp-aws-suite/named-servers.json delete mode 100644 docker/mcp-stdio-proxy/Dockerfile delete mode 100644 docker/mcp-stdio-proxy/README.md delete mode 100644 docker/mcp-stdio-proxy/named-servers.json delete mode 100644 docker/mcp-stdio-proxy/package.json delete mode 100644 docker/mcp-stdio-proxy/server.mjs delete mode 100644 docker/nginx/nginx.dev.conf delete mode 100644 docker/nginx/nginx.prod.conf delete mode 100644 docker/opensearch-dashboards.Dockerfile delete mode 100644 docker/opensearch-dashboards.prod.yml delete mode 100644 docker/opensearch-dashboards.yml delete mode 100755 docker/opensearch-init.sh delete mode 100644 docker/opensearch-security/action_groups.yml delete mode 100644 docker/opensearch-security/allowlist.yml delete mode 100644 docker/opensearch-security/audit.yml delete mode 100644 docker/opensearch-security/config.yml delete mode 100755 docker/opensearch-security/docker-entrypoint-security.sh delete mode 100644 docker/opensearch-security/internal_users.yml delete mode 100644 docker/opensearch-security/nodes_dn.yml delete mode 100644 docker/opensearch-security/roles.yml delete mode 100644 docker/opensearch-security/roles_mapping.yml delete mode 100644 docker/opensearch-security/tenants.yml delete mode 100644 docker/opensearch-security/whitelist.yml delete mode 100644 docker/opensearch.dev-secure.yml delete mode 100644 docker/redpanda-console-config.yaml delete mode 100755 docker/scripts/generate-certs.sh delete mode 100755 docker/scripts/hash-password.sh delete mode 100755 docker/scripts/security-init.sh delete mode 100644 install.sh delete mode 100755 scripts/db-reset-instance.sh delete mode 100755 scripts/instance-clean.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index c6fad0dae..000000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,340 +0,0 @@ -name: Release - -on: - push: - tags: - - 'v*.*.*' # Triggers on tags like v1.0.0, v1.2.3, etc. - workflow_dispatch: - inputs: - version: - description: 'Version tag (e.g., v1.0.0)' - required: true - type: string - -env: - REGISTRY: ghcr.io - IMAGE_PREFIX: ${{ github.repository_owner }}/studio - -jobs: - build-and-push: - name: Build and Push Docker Images - runs-on: ubuntu-latest - permissions: - contents: write - packages: write - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 # Full history for changelog generation - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Log in to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract version from tag - id: version - run: | - if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then - VERSION="${{ github.event.inputs.version }}" - else - VERSION="${GITHUB_REF#refs/tags/}" - fi - # Remove 'v' prefix if present for Docker tags - VERSION_CLEAN=$(echo "$VERSION" | sed 's/^v//') - echo "version=$VERSION" >> $GITHUB_OUTPUT - echo "version_clean=$VERSION_CLEAN" >> $GITHUB_OUTPUT - echo "Extracted version: $VERSION (clean: $VERSION_CLEAN)" - - - name: Determine if this is latest - id: is_latest - run: | - # Check if this is the latest tag (highest semver) - CURRENT_TAG="${{ steps.version.outputs.version_clean }}" - LATEST_TAG=$(git tag -l 'v*.*.*' | sort -V | tail -1 | sed 's/^v//') - - if [ "$CURRENT_TAG" = "$LATEST_TAG" ]; then - echo "is_latest=true" >> $GITHUB_OUTPUT - echo "This is the latest release" - else - echo "is_latest=false" >> $GITHUB_OUTPUT - echo "This is not the latest release (latest: $LATEST_TAG)" - fi - - - name: Get Git SHA - id: git_sha - run: echo "sha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT - - - name: Prepare image tags - id: tags - run: | - # Convert repository owner to lowercase for Docker registry compatibility - IMAGE_PREFIX_LOWER=$(echo "${{ env.IMAGE_PREFIX }}" | tr '[:upper:]' '[:lower:]') - VERSION_TAG="${{ env.REGISTRY }}/${IMAGE_PREFIX_LOWER}-backend:${{ steps.version.outputs.version_clean }}" - - if [ "${{ steps.is_latest.outputs.is_latest }}" = "true" ]; then - echo "backend_tags<> $GITHUB_OUTPUT - echo "$VERSION_TAG" >> $GITHUB_OUTPUT - echo "${{ env.REGISTRY }}/${IMAGE_PREFIX_LOWER}-backend:latest" >> $GITHUB_OUTPUT - echo "EOF" >> $GITHUB_OUTPUT - - echo "worker_tags<> $GITHUB_OUTPUT - echo "${{ env.REGISTRY }}/${IMAGE_PREFIX_LOWER}-worker:${{ steps.version.outputs.version_clean }}" >> $GITHUB_OUTPUT - echo "${{ env.REGISTRY }}/${IMAGE_PREFIX_LOWER}-worker:latest" >> $GITHUB_OUTPUT - echo "EOF" >> $GITHUB_OUTPUT - - echo "frontend_tags<> $GITHUB_OUTPUT - echo "${{ env.REGISTRY }}/${IMAGE_PREFIX_LOWER}-frontend:${{ steps.version.outputs.version_clean }}" >> $GITHUB_OUTPUT - echo "${{ env.REGISTRY }}/${IMAGE_PREFIX_LOWER}-frontend:latest" >> $GITHUB_OUTPUT - echo "EOF" >> $GITHUB_OUTPUT - else - echo "backend_tags<> $GITHUB_OUTPUT - echo "$VERSION_TAG" >> $GITHUB_OUTPUT - echo "EOF" >> $GITHUB_OUTPUT - - echo "worker_tags<> $GITHUB_OUTPUT - echo "${{ env.REGISTRY }}/${IMAGE_PREFIX_LOWER}-worker:${{ steps.version.outputs.version_clean }}" >> $GITHUB_OUTPUT - echo "EOF" >> $GITHUB_OUTPUT - - echo "frontend_tags<> $GITHUB_OUTPUT - echo "${{ env.REGISTRY }}/${IMAGE_PREFIX_LOWER}-frontend:${{ steps.version.outputs.version_clean }}" >> $GITHUB_OUTPUT - echo "EOF" >> $GITHUB_OUTPUT - fi - - - name: Build and push backend image - uses: docker/build-push-action@v5 - with: - context: . - file: ./Dockerfile - target: backend - push: true - platforms: linux/amd64,linux/arm64 - tags: ${{ steps.tags.outputs.backend_tags }} - build-args: | - VITE_GIT_SHA=${{ steps.git_sha.outputs.sha }} - POSTHOG_API_KEY=${{ secrets.POSTHOG_API_KEY }} - POSTHOG_HOST=${{ secrets.POSTHOG_HOST }} - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Build and push worker image - uses: docker/build-push-action@v5 - with: - context: . - file: ./Dockerfile - target: worker - push: true - platforms: linux/amd64,linux/arm64 - tags: ${{ steps.tags.outputs.worker_tags }} - build-args: | - VITE_GIT_SHA=${{ steps.git_sha.outputs.sha }} - POSTHOG_API_KEY=${{ secrets.POSTHOG_API_KEY }} - POSTHOG_HOST=${{ secrets.POSTHOG_HOST }} - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Build and push frontend image - uses: docker/build-push-action@v5 - with: - context: . - file: ./Dockerfile - target: frontend - push: true - platforms: linux/amd64,linux/arm64 - tags: ${{ steps.tags.outputs.frontend_tags }} - build-args: | - VITE_GIT_SHA=${{ steps.git_sha.outputs.sha }} - VITE_PUBLIC_POSTHOG_KEY=${{ secrets.POSTHOG_API_KEY }} - VITE_PUBLIC_POSTHOG_HOST=${{ secrets.POSTHOG_HOST }} - POSTHOG_API_KEY=${{ secrets.POSTHOG_API_KEY }} - POSTHOG_HOST=${{ secrets.POSTHOG_HOST }} - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Generate changelog - id: changelog - run: | - VERSION="${{ steps.version.outputs.version }}" - PREVIOUS_TAG=$(git describe --tags --abbrev=0 HEAD^ 2>/dev/null || echo "") - - # Convert repository owner to lowercase for Docker registry compatibility - IMAGE_PREFIX_LOWER=$(echo "${{ env.IMAGE_PREFIX }}" | tr '[:upper:]' '[:lower:]') - - if [ -z "$PREVIOUS_TAG" ]; then - echo "No previous tag found, generating full changelog" - CHANGELOG=$(git log --pretty=format:"- %s (%h)" --no-merges) - else - echo "Generating changelog from $PREVIOUS_TAG to $VERSION" - CHANGELOG=$(git log ${PREVIOUS_TAG}..HEAD --pretty=format:"- %s (%h)" --no-merges) - fi - - if [ -z "$CHANGELOG" ]; then - CHANGELOG="No changes detected" - fi - - { - echo "## Release $VERSION" - echo "" - echo "### Docker Images" - echo "" - echo "- Backend: \`${{ env.REGISTRY }}/${IMAGE_PREFIX_LOWER}-backend:${{ steps.version.outputs.version_clean }}\`" - echo "- Worker: \`${{ env.REGISTRY }}/${IMAGE_PREFIX_LOWER}-worker:${{ steps.version.outputs.version_clean }}\`" - echo "- Frontend: \`${{ env.REGISTRY }}/${IMAGE_PREFIX_LOWER}-frontend:${{ steps.version.outputs.version_clean }}\`" - echo "" - echo "### Changes" - echo "" - echo "$CHANGELOG" - } > CHANGELOG.md - - echo "changelog<> $GITHUB_OUTPUT - cat CHANGELOG.md >> $GITHUB_OUTPUT - echo "EOF" >> $GITHUB_OUTPUT - - - name: Create or update GitHub Release - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - VERSION="${{ steps.version.outputs.version }}" - IS_PRERELEASE="${{ contains(steps.version.outputs.version, '-') }}" - PRERELEASE_FLAG="" - if [ "$IS_PRERELEASE" = "true" ]; then - PRERELEASE_FLAG="--prerelease" - fi - - # Check if release already exists - if gh release view "$VERSION" --repo "${{ github.repository }}" > /dev/null 2>&1; then - echo "Release $VERSION already exists, updating..." - gh release edit "$VERSION" \ - --repo "${{ github.repository }}" \ - --notes-file CHANGELOG.md \ - $PRERELEASE_FLAG - else - echo "Creating new release $VERSION..." - gh release create "$VERSION" \ - --repo "${{ github.repository }}" \ - --title "Release $VERSION" \ - --notes-file CHANGELOG.md \ - $PRERELEASE_FLAG - fi - - - name: Update version check service - env: - VERSION_CHECK_URL: ${{ secrets.VERSION_CHECK_URL || 'https://version.shipsec.ai' }} - VERSION_CHECK_ADMIN_SECRET: ${{ secrets.VERSION_CHECK_ADMIN_SECRET }} - run: | - echo "🚀 Updating version check service..." - echo " App: studio" - echo " Version: ${{ steps.version.outputs.version_clean }}" - echo " Endpoint: $VERSION_CHECK_URL/api/admin/update-version" - - response=$(curl -s --connect-timeout 10 --max-time 30 -w "\n%{http_code}" -X POST "$VERSION_CHECK_URL/api/admin/update-version" \ - -H "Authorization: Bearer $VERSION_CHECK_ADMIN_SECRET" \ - -H "Content-Type: application/json" \ - -d "{\"app\":\"studio\",\"version\":\"${{ steps.version.outputs.version_clean }}\"}") - - http_code=$(echo "$response" | tail -n1) - body=$(echo "$response" | head -n-1) - - echo "Response: $body" - - if [ "$http_code" -eq 200 ]; then - echo "✅ Version check service updated successfully!" - else - echo "❌ Failed to update version check service (HTTP $http_code)" - echo "Response: $body" - exit 1 - fi - - - name: Verify version update - env: - VERSION_CHECK_URL: ${{ secrets.VERSION_CHECK_URL || 'https://version.shipsec.ai' }} - run: | - echo "🔍 Verifying version update..." - - # Wait a moment for cache to update - sleep 2 - - # Check if the version was updated - response=$(curl -s --connect-timeout 10 --max-time 30 "$VERSION_CHECK_URL/api/version/check?app=studio&version=0.0.1") - latest_version=$(echo "$response" | jq -r '.latest_version') - - echo "Latest version returned by API: $latest_version" - echo "Expected version: ${{ steps.version.outputs.version_clean }}" - - if [ "$latest_version" = "${{ steps.version.outputs.version_clean }}" ]; then - echo "✅ Verification successful! Version check service is returning the correct version." - else - echo "⚠️ Warning: API returned different version than expected" - echo " This might be expected if GitHub release fetching is enabled" - fi - - - name: Bump package.json version via PR - if: steps.is_latest.outputs.is_latest == 'true' - continue-on-error: true - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - VERSION_CLEAN="${{ steps.version.outputs.version_clean }}" - RUN_ID="${{ github.run_id }}" - BRANCH="chore/bump-version-${VERSION_CLEAN}-${RUN_ID}" - echo "Bumping root package.json version to ${VERSION_CLEAN}..." - - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - - git fetch origin main - - # Clean up any existing bump branch for this version - git push origin --delete "chore/bump-version-${VERSION_CLEAN}" 2>/dev/null || true - - git checkout -b "$BRANCH" origin/main - - # Update the root package.json version - jq --arg v "$VERSION_CLEAN" '.version = $v' package.json > package.json.tmp && mv package.json.tmp package.json - - # Only create PR if there's actually a change - if git diff --quiet package.json; then - echo "package.json already at version ${VERSION_CLEAN}, skipping." - else - git add package.json - git commit -m "chore: bump version to ${VERSION_CLEAN} [skip ci]" - git push origin "$BRANCH" - - # Create PR and enable auto-merge - PR_URL=$(gh pr create \ - --title "chore: bump version to ${VERSION_CLEAN}" \ - --body "Automated version bump from release workflow (${VERSION_CLEAN})." \ - --base main \ - --head "$BRANCH") - - echo "✅ Created PR: $PR_URL" - - # Attempt to enable auto-merge (requires repo setting enabled) - gh pr merge "$PR_URL" --auto --squash 2>/dev/null \ - && echo "✅ Auto-merge enabled" \ - || echo "⚠️ Auto-merge not available — merge the PR manually" - fi - - - name: Release summary - run: | - echo "📋 Release Summary:" - echo " ✅ Docker images built and pushed" - echo " ✅ GitHub Release created: ${{ steps.version.outputs.version }}" - echo " ✅ Version check service updated" - if [ "${{ steps.is_latest.outputs.is_latest }}" = "true" ]; then - echo " ✅ package.json version bump PR created" - else - echo " ⏭️ package.json NOT updated (not the latest version)" - fi - echo "" - echo "🎉 Users will now receive update notifications for version ${{ steps.version.outputs.version_clean }}" diff --git a/.gitignore b/.gitignore index 0b0e85af3..494695ba3 100644 --- a/.gitignore +++ b/.gitignore @@ -10,9 +10,6 @@ build/ *.local *.tsbuildinfo -# Context dumps (may contain secrets) -.context - # Environment variables .env .env.local @@ -21,7 +18,8 @@ docker/.env .env.development.local .env.test.local .env.production.local -.env.e2e +.env.eng-104 +.env.eng-104 .shipsec-instance # Logs @@ -75,3 +73,17 @@ vite.config.ts.timestamp-* .playground/ .omc/ MCP_FLOW_TRACE.md + +# Terraform / OpenTofu +.terraform/ +*.tfstate +*.tfstate.* +*.tfvars +*.tfvars.json +crash.log +crash.*.log +override.tf +override.tf.json +*_override.tf +*_override.tf.json +.terraform.lock.hcl.bak diff --git a/docker/PRODUCTION.md b/docker/PRODUCTION.md deleted file mode 100644 index dd5908d09..000000000 --- a/docker/PRODUCTION.md +++ /dev/null @@ -1,223 +0,0 @@ -# Production Deployment Guide - -This guide covers deploying the analytics infrastructure with security and SaaS multitenancy enabled. - -## Overview - -| Environment | Security | Multitenancy | Use Case | -|-------------|----------|--------------|----------| -| Development | Disabled | No | Local development, fast iteration | -| Production | Enabled | Yes (Strict) | Multi-tenant SaaS deployment | - -## SaaS Multitenancy Model - -**Key Principles:** -- Each customer gets complete data isolation by default -- No shared dashboards - sharing is explicitly opt-in -- Each customer has their own index pattern (`{customer_id}-*`) -- Tenants, roles, and users are created dynamically via backend - -**Index Naming Convention:** -``` -{customer_id}-analytics-* # Analytics data -{customer_id}-workflows-* # Workflow results -{customer_id}-scans-* # Scan results -``` - -## Quick Start (Production) - -```bash -# 1. Generate TLS certificates -./scripts/generate-certs.sh - -# 2. Set required environment variables -export OPENSEARCH_ADMIN_PASSWORD="your-secure-admin-password" -export OPENSEARCH_DASHBOARDS_PASSWORD="your-secure-dashboards-password" - -# 3. Start with production configuration -docker compose -f docker-compose.infra.yml -f docker-compose.prod.yml up -d -``` - -## Files Overview - -| File | Purpose | -|------|---------| -| `docker-compose.infra.yml` | Base infrastructure (dev mode, PM2 on host) | -| `docker-compose.full.yml` | Full stack containerized (simple prod, no security) | -| `docker-compose.prod.yml` | Security overlay (combines with infra.yml for SaaS) | -| `nginx/nginx.dev.conf` | Nginx routing to host (PM2 services) | -| `nginx/nginx.prod.conf` | Nginx routing to containers | -| `opensearch-dashboards.yml` | Dashboards config (dev) | -| `opensearch-dashboards.prod.yml` | Dashboards config (prod with multitenancy) | -| `scripts/generate-certs.sh` | TLS certificate generator | -| `opensearch-security/` | Security plugin configuration | -| `certs/` | Generated certificates (gitignored) | - -See [README.md](README.md) for detailed usage of each compose file. - -## Customer Provisioning (Backend Integration) - -When a new customer is onboarded, the backend must create: - -### 1. Create Customer Tenant -```bash -PUT /_plugins/_security/api/tenants/{customer_id} -{ - "description": "Tenant for customer {customer_id}" -} -``` - -### 2. Create Customer Role (with Index Isolation) -```bash -PUT /_plugins/_security/api/roles/customer_{customer_id}_rw -{ - "cluster_permissions": ["cluster_composite_ops_ro"], - "index_permissions": [{ - "index_patterns": ["{customer_id}-*"], - "allowed_actions": ["read", "write", "create_index", "indices:data/read/*", "indices:data/write/*"] - }], - "tenant_permissions": [{ - "tenant_patterns": ["{customer_id}"], - "allowed_actions": ["kibana_all_write"] - }] -} -``` - -### 3. Create Customer User -```bash -PUT /_plugins/_security/api/internalusers/{user_email} -{ - "password": "hashed_password", - "backend_roles": ["customer_{customer_id}"], - "attributes": { - "customer_id": "{customer_id}", - "email": "{user_email}" - } -} -``` - -### 4. Map User to Role -```bash -PUT /_plugins/_security/api/rolesmapping/customer_{customer_id}_rw -{ - "users": ["{user_email}"], - "backend_roles": ["customer_{customer_id}"] -} -``` - -## Security Configuration - -### TLS Certificates - -The `scripts/generate-certs.sh` script generates: - -- **root-ca.pem** - Root certificate authority -- **node.pem / node-key.pem** - OpenSearch node certificate -- **admin.pem / admin-key.pem** - Admin certificate for cluster management - -For production: -- Use a proper CA (Let's Encrypt, internal PKI) -- Store private keys in a secrets manager (Vault, AWS Secrets Manager) -- Set up certificate rotation before expiration - -### System Users - -Only two system users are defined (in `internal_users.yml`): - -| User | Purpose | -|------|---------| -| `admin` | Platform operations - DO NOT give to customers | -| `kibanaserver` | Dashboards backend communication | - -Customer users are created dynamically via the Security REST API. - -### Password Hashing - -Generate password hashes for users: -```bash -docker run -it opensearchproject/opensearch:2.11.1 \ - /usr/share/opensearch/plugins/opensearch-security/tools/hash.sh -p YOUR_PASSWORD -``` - -## Data Isolation Verification - -After setting up a customer, verify isolation: - -```bash -# As customer user - should only see their data -curl -u user@customer.com:password \ - "https://localhost:9200/{customer_id}-*/_search" - -# Should NOT be able to access other customer's data (403 Forbidden) -curl -u user@customer.com:password \ - "https://localhost:9200/other_customer-*/_search" -``` - -## Environment Variables - -| Variable | Required | Description | -|----------|----------|-------------| -| `OPENSEARCH_ADMIN_PASSWORD` | Yes | Admin user password | -| `OPENSEARCH_DASHBOARDS_PASSWORD` | Yes | kibanaserver user password | - -## Updating Security Configuration - -After modifying security files, apply changes: - -```bash -docker exec -it shipsec-opensearch \ - /usr/share/opensearch/plugins/opensearch-security/tools/securityadmin.sh \ - -cd /usr/share/opensearch/config/opensearch-security \ - -icl -nhnv \ - -cacert /usr/share/opensearch/config/certs/root-ca.pem \ - -cert /usr/share/opensearch/config/certs/admin.pem \ - -key /usr/share/opensearch/config/certs/admin-key.pem -``` - -## Troubleshooting - -### Container fails to start - -Check logs: -```bash -docker logs shipsec-opensearch -docker logs shipsec-opensearch-dashboards -``` - -Common issues: -- Certificate permissions (should be 600 for keys, 644 for certs) -- Missing environment variables -- Incorrect certificate paths - -### Cannot connect to secured cluster - -```bash -# Test with curl -curl -k -u admin:PASSWORD https://localhost:9200/_cluster/health -``` - -### Customer cannot see their dashboards - -1. Verify tenant was created for customer -2. Check user has correct backend_roles -3. Verify role has correct tenant_permissions -4. Check index pattern matches customer's indices - -### Cross-tenant data leak - -If a customer can see another customer's data: -1. Verify index_patterns in role are correctly scoped to `{customer_id}-*` -2. Check role mapping is correct -3. Ensure user's backend_roles match their customer ID - -## Switching Between Environments - -**Development (no security):** -```bash -docker compose -f docker-compose.infra.yml up -d -``` - -**Production (with security):** -```bash -docker compose -f docker-compose.infra.yml -f docker-compose.prod.yml up -d -``` diff --git a/docker/README.md b/docker/README.md deleted file mode 100644 index d470d1b96..000000000 --- a/docker/README.md +++ /dev/null @@ -1,169 +0,0 @@ -# Docker Configuration - -This directory contains Docker Compose configurations for running ShipSec Studio in different environments. - -## Docker Compose Files - -| File | Purpose | When to Use | -| -------------------------- | ---------------------------- | ------------------------------------------------------ | -| `docker-compose.infra.yml` | Infrastructure services only | Development with PM2 (frontend/backend on host) | -| `docker-compose.full.yml` | Full stack in containers | Self-hosted deployment, all services containerized | -| `docker-compose.prod.yml` | Security overlay | Production SaaS with multitenancy (overlays infra.yml) | - -## Environment Modes - -### Development Mode (`just dev`) - -```bash -just dev -``` - -- **Compose file**: `docker-compose.infra.yml` -- **Frontend/Backend**: Run via PM2 on host machine -- **Infrastructure**: Runs in Docker (Postgres, Redis, Temporal, OpenSearch, etc.) -- **Nginx**: Uses `nginx.dev.conf` pointing to `host.docker.internal` -- **Security**: Disabled for fast iteration - -**Access (all via port 80):** - -- Frontend: http://localhost/ -- Backend API: http://localhost/api/ -- Analytics: http://localhost/analytics/ - -**Nginx Routing (nginx.dev.conf):** - -| Path | Target (host machine) | Port | -| -------------- | ----------------------------- | ---- | -| `/analytics/*` | opensearch-dashboards | 5601 | -| `/api/*` | host.docker.internal:backend | 3211 | -| `/*` | host.docker.internal:frontend | 5173 | - -> **Note:** Service ports (5173, 3211, 5601) are accessible directly for debugging but should not be used in normal development. All traffic flows through nginx on port 80. - -### Production Mode (`just prod`) - -```bash -just prod -``` - -- **Compose file**: `docker-compose.full.yml` -- **All services**: Run as Docker containers -- **Nginx**: Unified entry point on port 80 -- **Security**: Disabled (simple deployment) - -**Access (all via port 80):** - -- Frontend: http://localhost/ -- Backend API: http://localhost/api/ -- Analytics: http://localhost/analytics/ - -**Nginx Routing (nginx.prod.conf):** - -| Path | Target Container | Port | -| -------------- | --------------------- | ---- | -| `/analytics/*` | opensearch-dashboards | 5601 | -| `/api/*` | backend | 3211 | -| `/*` | frontend | 8080 | - -> **Note:** Frontend and backend containers only expose ports internally. All external traffic flows through nginx on port 80. - -### Production Secure Mode (`just prod-secure`) - -```bash -just generate-certs -export OPENSEARCH_ADMIN_PASSWORD='secure-password' -export OPENSEARCH_DASHBOARDS_PASSWORD='secure-password' -just prod-secure -``` - -- **Compose files**: `docker-compose.infra.yml` + `docker-compose.prod.yml` (overlay) -- **Security**: TLS enabled, authentication required -- **Multitenancy**: Strict SaaS isolation per customer -- **Nginx**: Uses `nginx.prod.conf` with container networking - -**Access:** - -- Analytics: https://localhost/analytics (auth required) -- OpenSearch: https://localhost:9200 (TLS) - -## Nginx Configuration - -| File | Target Services | Use Case | -| ----------------------- | ------------------------------------------------------------- | ---------------------------------------- | -| `nginx/nginx.dev.conf` | `host.docker.internal:5173/3211` | Dev (PM2 on host) | -| `nginx/nginx.prod.conf` | `frontend:8080`, `backend:3211`, `opensearch-dashboards:5601` | Container mode (full stack + production) | - -### Routing Architecture - -All modes use nginx as a reverse proxy with unified routing: - -``` -┌─────────────────────────────────────────────────┐ -│ Nginx (port 80/443) │ -├─────────────────────────────────────────────────┤ -│ /analytics/* → OpenSearch Dashboards:5601 │ -│ /api/* → Backend:3211 │ -│ /* → Frontend:8080 │ -└─────────────────────────────────────────────────┘ -``` - -### OpenSearch Dashboards BasePath - -OpenSearch Dashboards is configured with `server.basePath: "/analytics"` to work behind nginx: - -- Incoming requests: `/analytics/app/discover` → internally processed as `/app/discover` -- Outgoing URLs: Automatically prefixed with `/analytics` - -## Analytics Pipeline - -The worker service writes analytics data to OpenSearch via the Analytics Sink component. - -**Required Environment Variable:** - -```yaml -OPENSEARCH_URL=http://opensearch:9200 -``` - -This is pre-configured in `docker-compose.full.yml`. For detailed analytics documentation, see [docs/analytics.md](../docs/analytics.md). - -## Directory Structure - -``` -docker/ -├── docker-compose.infra.yml # Infrastructure (dev base) -├── docker-compose.full.yml # Full stack containerized -├── docker-compose.prod.yml # Security overlay for prod -├── nginx/ -│ ├── nginx.dev.conf # Routes to host (PM2) -│ └── nginx.prod.conf # Routes to containers -├── opensearch-dashboards.yml # Dev dashboards config -├── opensearch-dashboards.prod.yml # Prod dashboards config -├── opensearch-security/ # Security plugin configs -│ ├── internal_users.yml -│ ├── roles.yml -│ ├── roles_mapping.yml -│ └── tenants.yml -├── scripts/ -│ └── generate-certs.sh # TLS certificate generator -├── certs/ # Generated certs (gitignored) -├── PRODUCTION.md # Production deployment guide -└── README.md # This file -``` - -## Quick Reference - -| Command | Description | -| --------------------- | ------------------------------------------ | -| `just dev` | Start dev environment (PM2 + Docker infra) | -| `just dev stop` | Stop dev environment | -| `just prod` | Start full stack in Docker | -| `just prod stop` | Stop production | -| `just prod-secure` | Start with security & multitenancy | -| `just generate-certs` | Generate TLS certificates | -| `just infra up` | Start infrastructure only | -| `just help` | Show all available commands | - -## See Also - -- [PRODUCTION.md](PRODUCTION.md) - Detailed production deployment and customer provisioning guide -- [docs/analytics.md](../docs/analytics.md) - Analytics pipeline and OpenSearch configuration diff --git a/docker/SECURE-DEV-MODE.md b/docker/SECURE-DEV-MODE.md deleted file mode 100644 index f20acccbc..000000000 --- a/docker/SECURE-DEV-MODE.md +++ /dev/null @@ -1,126 +0,0 @@ -# Secure Development Mode - -This document describes the secure development environment setup with OpenSearch Security enabled for multi-tenant isolation. - -## Overview - -The `just dev` command now starts the development environment with full OpenSearch Security enabled, matching the production security model. This provides: - -- **TLS encryption** for all OpenSearch communication -- **Multi-tenant isolation** - each organization's data is isolated -- **Authentication required** - no anonymous access -- **Same security model as production** - test security features locally - -## Quick Start - -```bash -# Start secure dev environment (recommended) -just dev - -# Start without security (faster, for quick iteration) -just dev-insecure -``` - -## Architecture - -### Docker Compose Files - -| File | Purpose | -|------|---------| -| `docker-compose.infra.yml` | Base infrastructure (Postgres, Redis, Temporal, etc.) | -| `docker-compose.dev-secure.yml` | Security overlay for development | -| `docker-compose.prod.yml` | Production security configuration | - -### Security Configuration Files - -Located in `docker/opensearch-security/`: - -| File | Purpose | -|------|---------| -| `config.yml` | Authentication/authorization backends (proxy auth) | -| `internal_users.yml` | System users (admin, kibanaserver, worker) | -| `roles.yml` | Role definitions with index permissions | -| `roles_mapping.yml` | User-to-role mappings | -| `action_groups.yml` | Permission groups for roles | -| `tenants.yml` | Tenant definitions | -| `audit.yml` | Audit logging configuration | - -### TLS Certificates - -Certificates are auto-generated on first run and stored in `docker/certs/`: - -- `root-ca.pem` / `root-ca-key.pem` - Certificate Authority -- `admin.pem` / `admin-key.pem` - Admin certificate for securityadmin tool -- `node.pem` / `node-key.pem` - OpenSearch node certificate - -## Default Credentials - -For development convenience, default passwords are set: - -| User | Password | Purpose | -|------|----------|---------| -| `admin` | `admin` | Platform administrator | -| `kibanaserver` | `admin` | Dashboards backend communication | -| `worker` | `admin` | Worker service for indexing | - -**Important**: Change these in production via environment variables: -- `OPENSEARCH_ADMIN_PASSWORD` -- `OPENSEARCH_DASHBOARDS_PASSWORD` - -## Multi-Tenant Isolation - -### How It Works - -1. **Index Pattern**: Each organization's data is stored in indices prefixed with their org ID: - - `security-findings-{org_id}-*` - -2. **Tenant Isolation**: OpenSearch Dashboards uses tenants to isolate saved objects (dashboards, visualizations) - -3. **Role-Based Access**: Dynamic roles are created per customer restricting access to their indices only - -### Dynamic Provisioning - -When a new customer is onboarded, the backend creates: -1. A tenant for their organization -2. A role with permissions scoped to their indices -3. User-to-role mappings - -## Troubleshooting - -### Check Container Health - -```bash -just dev status -docker logs shipsec-opensearch -docker logs shipsec-opensearch-dashboards -``` - -### Reset Security Configuration - -```bash -# Clean everything and restart -just dev clean && just dev - -# Or manually run securityadmin -docker exec shipsec-opensearch /usr/share/opensearch/plugins/opensearch-security/tools/securityadmin.sh \ - -cd /usr/share/opensearch/config/opensearch-security \ - -icl -nhnv \ - -cacert /usr/share/opensearch/config/certs/root-ca.pem \ - -cert /usr/share/opensearch/config/certs/admin.pem \ - -key /usr/share/opensearch/config/certs/admin-key.pem -``` - -### Regenerate Certificates - -```bash -rm -rf docker/certs -just generate-certs -just dev clean && just dev -``` - -## Changes from Previous Setup - -1. **`just dev`** now runs with security enabled (was insecure) -2. **`just dev-insecure`** is the new command for fast, insecure development -3. Certificates are auto-generated if missing -4. Environment variable `OPENSEARCH_SECURITY_ENABLED=true` is set for backend/worker diff --git a/docker/certs/.gitignore b/docker/certs/.gitignore deleted file mode 100644 index 5ae618b73..000000000 --- a/docker/certs/.gitignore +++ /dev/null @@ -1,7 +0,0 @@ -# Ignore generated certificates and private keys -# These should NEVER be committed to version control -*.pem -*.key -*.crt -*.csr -*.srl diff --git a/docker/docker-compose.dev-ports.yml b/docker/docker-compose.dev-ports.yml deleted file mode 100644 index 0da3bd056..000000000 --- a/docker/docker-compose.dev-ports.yml +++ /dev/null @@ -1,54 +0,0 @@ -# Development Ports Overlay -# -# WARNING: These ports bypass ALL nginx authentication! -# Use ONLY for local development where direct service access is needed. -# -# Usage: -# docker compose -f docker-compose.infra.yml -f docker-compose.dev-ports.yml up -d -# -# This overlay exposes all service ports on loopback (127.0.0.1) only, -# preventing external network access while allowing local development tools to connect. - -services: - postgres: - ports: - - "127.0.0.1:5433:5432" - - temporal: - ports: - - "127.0.0.1:7233:7233" - - temporal-ui: - ports: - - "127.0.0.1:8081:8080" - - minio: - ports: - - "127.0.0.1:9000:9000" - - "127.0.0.1:9001:9001" - - redis: - ports: - - "127.0.0.1:6379:6379" - - loki: - ports: - - "127.0.0.1:3100:3100" - - redpanda: - ports: - - "127.0.0.1:9092:9092" - - "127.0.0.1:9644:9644" - - redpanda-console: - ports: - - "127.0.0.1:8082:8080" - - opensearch: - ports: - - "127.0.0.1:9200:9200" - - "127.0.0.1:9600:9600" - - opensearch-dashboards: - ports: - - "127.0.0.1:5601:5601" diff --git a/docker/docker-compose.dev-secure.yml b/docker/docker-compose.dev-secure.yml deleted file mode 100644 index 1411e3822..000000000 --- a/docker/docker-compose.dev-secure.yml +++ /dev/null @@ -1,75 +0,0 @@ -# Development Docker Compose with Security & Multitenancy -# -# Usage: -# docker compose -f docker-compose.infra.yml -f docker-compose.dev-secure.yml up -d -# -# This overlay enables OpenSearch Security plugin for development: -# - TLS encryption -# - Multi-tenant isolation -# - Same security model as production -# -# Requires: -# 1. TLS certificates in docker/certs/ (run: just generate-certs) -# 2. Environment variables (auto-set by just dev for convenience): -# - OPENSEARCH_ADMIN_PASSWORD -# - OPENSEARCH_DASHBOARDS_PASSWORD - -services: - opensearch: - # Custom entrypoint for proxy auth config templating - entrypoint: ["/usr/share/opensearch/config/opensearch-security/docker-entrypoint-security.sh"] - environment: - # Enable security plugin (override infra.yml settings) - - DISABLE_SECURITY_PLUGIN=false - - DISABLE_INSTALL_DEMO_CONFIG=true - # Admin password for healthcheck (default: admin) - - OPENSEARCH_ADMIN_PASSWORD=${OPENSEARCH_ADMIN_PASSWORD:-admin} - # Proxy auth - trusted proxy IP regex (Docker networks: 172.x, 192.168.x, 10.x) - # NOTE: Double backslashes required - sed consumes one level during config templating - - OPENSEARCH_INTERNAL_PROXIES=(172|192|10)\\.\\d+\\.\\d+\\.\\d+ - volumes: - - opensearch_data:/usr/share/opensearch/data - - ./certs:/usr/share/opensearch/config/certs:ro - - ./opensearch-security:/usr/share/opensearch/config/opensearch-security:ro - # Custom config file with admin_dn as YAML array (env vars don't support arrays) - - ./opensearch.dev-secure.yml:/usr/share/opensearch/config/opensearch.yml:ro - healthcheck: - test: ["CMD-SHELL", "curl -sf -u admin:$${OPENSEARCH_ADMIN_PASSWORD:-admin} --cacert /usr/share/opensearch/config/certs/root-ca.pem https://localhost:9200/_cluster/health || exit 1"] - interval: 30s - timeout: 10s - retries: 10 - start_period: 60s - - opensearch-dashboards: - environment: - # Enable security plugin (override infra.yml settings) - - DISABLE_SECURITY_DASHBOARDS_PLUGIN=false - - OPENSEARCH_HOSTS=["https://opensearch:9200"] - - OPENSEARCH_DASHBOARDS_PASSWORD=${OPENSEARCH_DASHBOARDS_PASSWORD:-admin} - volumes: - - ./opensearch-dashboards.prod.yml:/usr/share/opensearch-dashboards/config/opensearch_dashboards.yml:ro - - ./certs:/usr/share/opensearch-dashboards/config/certs:ro - healthcheck: - # Check if server responds (401 is fine - means server is up, security just requires auth) - test: ["CMD-SHELL", "curl -sf -o /dev/null -w '%{http_code}' http://localhost:5601/analytics/api/status | grep -qE '(200|401)' || exit 1"] - interval: 30s - timeout: 10s - retries: 10 - start_period: 60s - - # Override init script to work with secured cluster - opensearch-init: - environment: - - OPENSEARCH_SECURITY_ENABLED=true - - OPENSEARCH_ADMIN_PASSWORD=${OPENSEARCH_ADMIN_PASSWORD:-admin} - - OPENSEARCH_CA_CERT=/certs/root-ca.pem - volumes: - - ./opensearch-init.sh:/init.sh:ro - - ./certs:/certs:ro - -volumes: - opensearch_data: - -networks: - default: - name: shipsec-network diff --git a/docker/docker-compose.full.yml b/docker/docker-compose.full.yml deleted file mode 100644 index 48314ea04..000000000 --- a/docker/docker-compose.full.yml +++ /dev/null @@ -1,415 +0,0 @@ -services: - # Infrastructure - postgres: - image: postgres:16-alpine - container_name: shipsec-postgres - environment: - POSTGRES_USER: shipsec - POSTGRES_PASSWORD: shipsec - POSTGRES_DB: shipsec - POSTGRES_MULTIPLE_DATABASES: temporal - # Internal only - no direct port access in production - expose: - - '5432' - volumes: - - postgres_data:/var/lib/postgresql/data - - ./init-db:/docker-entrypoint-initdb.d - healthcheck: - test: ['CMD-SHELL', 'pg_isready -U shipsec'] - interval: 5s - timeout: 3s - retries: 10 - restart: unless-stopped - - temporal: - image: temporalio/auto-setup:latest - container_name: shipsec-temporal - depends_on: - postgres: - condition: service_healthy - environment: - - DB=postgres12 - - DB_PORT=5432 - - DB_NAME=temporal - - POSTGRES_USER=shipsec - - POSTGRES_PWD=shipsec - - POSTGRES_SEEDS=postgres - - AUTO_SETUP=true - expose: - - '7233' - volumes: - - temporal_data:/var/lib/temporal - restart: unless-stopped - healthcheck: - test: ['CMD-SHELL', 'tctl --address $(hostname -i):7233 cluster health'] - interval: 30s - timeout: 10s - retries: 5 - - temporal-ui: - image: temporalio/ui:latest - container_name: shipsec-temporal-ui - environment: - - TEMPORAL_ADDRESS=temporal:7233 - - TEMPORAL_NAMESPACE=default - expose: - - '8080' - depends_on: - - temporal - restart: unless-stopped - healthcheck: - test: ['CMD', 'curl', '-sf', 'http://localhost:8080'] - interval: 30s - timeout: 10s - retries: 5 - - minio: - image: minio/minio:RELEASE.2024-10-02T17-50-41Z - container_name: shipsec-minio - command: server /data --console-address ":9001" - environment: - MINIO_ROOT_USER: minioadmin - MINIO_ROOT_PASSWORD: minioadmin - expose: - - '9000' - - '9001' - volumes: - - minio_data:/data - restart: unless-stopped - healthcheck: - test: ['CMD', 'curl', '-f', 'http://localhost:9000/minio/health/live'] - interval: 30s - timeout: 10s - retries: 5 - - loki: - image: grafana/loki:3.2.1 - container_name: shipsec-loki - command: -config.file=/etc/loki/local-config.yaml - expose: - - '3100' - volumes: - - ./loki/loki-config.yaml:/etc/loki/local-config.yaml - - loki_data:/loki - restart: unless-stopped - healthcheck: - test: ['CMD', 'wget', '--no-verbose', '--tries=1', '--spider', 'http://localhost:3100/ready'] - interval: 30s - timeout: 10s - retries: 5 - - redis: - image: redis:latest - container_name: shipsec-redis - expose: - - '6379' - volumes: - - redis_data:/data - restart: unless-stopped - healthcheck: - test: ['CMD', 'redis-cli', 'ping'] - interval: 30s - timeout: 10s - retries: 5 - - redpanda: - image: redpandadata/redpanda:v24.2.5 - container_name: shipsec-redpanda - command: - - redpanda - - start - - --mode=dev-container - - --smp=1 - - --reserve-memory=0M - - --overprovisioned - - --node-id=0 - - --check=false - - --advertise-kafka-addr=redpanda:9092 - expose: - - '9092' - - '9644' - volumes: - - redpanda_data:/var/lib/redpanda/data - restart: unless-stopped - healthcheck: - test: ['CMD', 'curl', '-f', 'http://localhost:9644/v1/status/ready'] - interval: 30s - timeout: 10s - retries: 5 - - redpanda-console: - image: redpandadata/console:v2.7.2 - container_name: shipsec-redpanda-console - depends_on: - - redpanda - environment: - CONFIG_FILEPATH: /etc/redpanda/console-config.yaml - expose: - - '8080' - volumes: - - ./redpanda-console-config.yaml:/etc/redpanda/console-config.yaml:ro - restart: unless-stopped - - opensearch: - image: opensearchproject/opensearch:2.11.1 - container_name: shipsec-opensearch - environment: - - discovery.type=single-node - - bootstrap.memory_lock=true - - 'OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m' - - DISABLE_SECURITY_PLUGIN=true - - DISABLE_INSTALL_DEMO_CONFIG=true - ulimits: - memlock: - soft: -1 - hard: -1 - nofile: - soft: 65536 - hard: 65536 - expose: - - '9200' - - '9600' - volumes: - - opensearch_data:/usr/share/opensearch/data - restart: unless-stopped - healthcheck: - test: ['CMD-SHELL', 'curl -f http://localhost:9200/_cluster/health || exit 1'] - interval: 30s - timeout: 10s - retries: 5 - - opensearch-dashboards: - build: - context: . - dockerfile: opensearch-dashboards.Dockerfile - image: shipsec-opensearch-dashboards:2.11.1 - container_name: shipsec-opensearch-dashboards - depends_on: - opensearch: - condition: service_healthy - environment: - - OPENSEARCH_HOSTS=["http://opensearch:9200"] - - DISABLE_SECURITY_DASHBOARDS_PLUGIN=true - expose: - - '5601' - volumes: - - ./opensearch-dashboards.yml:/usr/share/opensearch-dashboards/config/opensearch_dashboards.yml:ro - restart: unless-stopped - healthcheck: - test: ['CMD-SHELL', 'curl -f http://localhost:5601/analytics/api/status || exit 1'] - interval: 30s - timeout: 10s - retries: 5 - - opensearch-init: - image: curlimages/curl:8.5.0 - container_name: shipsec-opensearch-init - depends_on: - opensearch-dashboards: - condition: service_healthy - volumes: - - ./opensearch-init.sh:/init.sh:ro - entrypoint: ['/bin/sh', '/init.sh'] - restart: 'no' - - # Applications - dind: - image: docker:27-dind - container_name: shipsec-dind - privileged: true - command: ['--host=tcp://0.0.0.0:2375', '--storage-driver=overlay2'] - environment: - - DOCKER_TLS_CERTDIR= - volumes: - - docker_data:/var/lib/docker - healthcheck: - test: ['CMD', 'docker', 'info'] - interval: 30s - timeout: 10s - retries: 5 - restart: unless-stopped - - backend: - image: ghcr.io/shipsecai/studio-backend:${SHIPSEC_TAG:-latest} - build: - context: .. - dockerfile: Dockerfile - target: backend - container_name: shipsec-backend - environment: - - NODE_ENV=production - - SHIPSEC_ENV=production - - PORT=3211 - - DATABASE_URL=postgresql://shipsec:shipsec@postgres:5432/shipsec - - ENABLE_INGEST_SERVICES=true - - TEMPORAL_ADDRESS=temporal:7233 - - TEMPORAL_NAMESPACE=shipsec-prod - - TEMPORAL_TASK_QUEUE=shipsec-prod - - MINIO_ROOT_USER=minioadmin - - MINIO_ROOT_PASSWORD=minioadmin - - MINIO_ENDPOINT=minio - - MINIO_PORT=9000 - - LOKI_URL=http://loki:3100 - - TERMINAL_REDIS_URL=redis://redis:6379 - - LOG_KAFKA_BROKERS=redpanda:9092 - - LOG_KAFKA_TOPIC=telemetry.logs - - LOG_KAFKA_CLIENT_ID=shipsec-backend - - LOG_KAFKA_GROUP_ID=shipsec-backend-log-consumer - - EVENT_KAFKA_TOPIC=telemetry.events - - EVENT_KAFKA_CLIENT_ID=shipsec-backend-events - - EVENT_KAFKA_GROUP_ID=shipsec-event-ingestor - - AUTH_PROVIDER=${AUTH_PROVIDER:-local} - - CLERK_PUBLISHABLE_KEY=${CLERK_PUBLISHABLE_KEY:-} - - CLERK_SECRET_KEY=${CLERK_SECRET_KEY:-} - - SESSION_SECRET=${SESSION_SECRET:-} - # Set to 'true' to disable analytics - - DISABLE_ANALYTICS=${DISABLE_ANALYTICS:-false} - # Internal service token for worker->backend auth - - INTERNAL_SERVICE_TOKEN=${INTERNAL_SERVICE_TOKEN:-} - # OpenSearch tenant provisioning - - OPENSEARCH_SECURITY_ENABLED=${OPENSEARCH_SECURITY_ENABLED:-false} - - OPENSEARCH_URL=http://opensearch:9200 - - OPENSEARCH_DASHBOARDS_URL=http://opensearch-dashboards:5601/analytics - - OPENSEARCH_ADMIN_USERNAME=${OPENSEARCH_ADMIN_USERNAME:-admin} - - OPENSEARCH_ADMIN_PASSWORD=${OPENSEARCH_ADMIN_PASSWORD:-} - # Internal only - accessed via nginx at /api/ - expose: - - '3211' - depends_on: - postgres: - condition: service_healthy - temporal: - condition: service_started - minio: - condition: service_healthy - redis: - condition: service_healthy - restart: unless-stopped - - frontend: - image: ghcr.io/shipsecai/studio-frontend:${SHIPSEC_TAG:-latest} - build: - context: .. - dockerfile: Dockerfile - target: frontend - args: - VITE_API_URL: ${VITE_API_URL:-http://localhost} - VITE_BACKEND_URL: ${VITE_BACKEND_URL:-http://localhost} - VITE_AUTH_PROVIDER: ${VITE_AUTH_PROVIDER:-local} - VITE_DEFAULT_ORG_ID: ${VITE_DEFAULT_ORG_ID:-local-dev} - VITE_CLERK_PUBLISHABLE_KEY: ${VITE_CLERK_PUBLISHABLE_KEY:-} - VITE_GIT_SHA: ${GIT_SHA:-unknown} - VITE_PUBLIC_POSTHOG_KEY: ${VITE_PUBLIC_POSTHOG_KEY:-} - VITE_PUBLIC_POSTHOG_HOST: ${VITE_PUBLIC_POSTHOG_HOST:-} - VITE_OPENSEARCH_DASHBOARDS_URL: ${VITE_OPENSEARCH_DASHBOARDS_URL:-/analytics} - container_name: shipsec-frontend - environment: - - VITE_API_URL=${VITE_API_URL:-http://localhost} - - VITE_BACKEND_URL=${VITE_BACKEND_URL:-http://localhost} - - VITE_AUTH_PROVIDER=${VITE_AUTH_PROVIDER:-local} - - VITE_DEFAULT_ORG_ID=${VITE_DEFAULT_ORG_ID:-local-dev} - - VITE_CLERK_PUBLISHABLE_KEY=${VITE_CLERK_PUBLISHABLE_KEY:-} - - VITE_OPENSEARCH_DASHBOARDS_URL=${VITE_OPENSEARCH_DASHBOARDS_URL:-/analytics} - # Internal only - accessed via nginx at / - expose: - - '8080' - depends_on: - - backend - restart: unless-stopped - healthcheck: - test: ['CMD', 'curl', '-sf', 'http://localhost:8080'] - interval: 30s - timeout: 10s - retries: 5 - - worker: - image: ghcr.io/shipsecai/studio-worker:${SHIPSEC_TAG:-latest} - build: - context: .. - dockerfile: Dockerfile - target: worker - container_name: shipsec-worker - environment: - - NODE_ENV=production - - SHIPSEC_ENV=production - - DATABASE_URL=postgresql://shipsec:shipsec@postgres:5432/shipsec - - ENABLE_INGEST_SERVICES=true - - TEMPORAL_ADDRESS=temporal:7233 - - TEMPORAL_NAMESPACE=shipsec-prod - - TEMPORAL_TASK_QUEUE=shipsec-prod - - MINIO_ROOT_USER=minioadmin - - MINIO_ROOT_PASSWORD=minioadmin - - MINIO_ENDPOINT=minio - - MINIO_PORT=9000 - - LOKI_URL=http://loki:3100 - - TERMINAL_REDIS_URL=redis://redis:6379 - - DOCKER_HOST=tcp://dind:2375 - - LOG_KAFKA_BROKERS=redpanda:9092 - - LOG_KAFKA_TOPIC=telemetry.logs - - LOG_KAFKA_CLIENT_ID=shipsec-worker - - EVENT_KAFKA_TOPIC=telemetry.events - - EVENT_KAFKA_CLIENT_ID=shipsec-worker-events - # OpenSearch for Analytics Sink - - OPENSEARCH_URL=http://opensearch:9200 - - OPENSEARCH_DASHBOARDS_URL=http://opensearch-dashboards:5601/analytics - # Tenant provisioning (for multi-tenant security mode) - - BACKEND_URL=http://backend:3211 - - INTERNAL_SERVICE_TOKEN=${INTERNAL_SERVICE_TOKEN:-} - - OPENSEARCH_SECURITY_ENABLED=${OPENSEARCH_SECURITY_ENABLED:-false} - depends_on: - postgres: - condition: service_healthy - temporal: - condition: service_started - minio: - condition: service_healthy - dind: - condition: service_healthy - redis: - condition: service_healthy - redpanda: - condition: service_healthy - opensearch: - condition: service_healthy - restart: unless-stopped - healthcheck: - test: ['CMD', 'node', '-e', 'process.exit(0)'] - interval: 30s - timeout: 10s - retries: 5 - - # Nginx reverse proxy - unified entry point - nginx: - image: nginx:1.25-alpine - container_name: shipsec-nginx - depends_on: - frontend: - condition: service_healthy - backend: - condition: service_started - opensearch-dashboards: - condition: service_healthy - ports: - - '80:80' - volumes: - - ./nginx/nginx.prod.conf:/etc/nginx/nginx.conf:ro - restart: unless-stopped - healthcheck: - test: ['CMD', 'curl', '-sf', 'http://localhost/health'] - interval: 30s - timeout: 10s - retries: 5 - -volumes: - postgres_data: - minio_data: - loki_data: - temporal_data: - docker_data: - redis_data: - redpanda_data: - opensearch_data: - -networks: - default: - name: shipsec-network diff --git a/docker/docker-compose.infra.yml b/docker/docker-compose.infra.yml deleted file mode 100644 index 1e9e619c9..000000000 --- a/docker/docker-compose.infra.yml +++ /dev/null @@ -1,242 +0,0 @@ -services: - postgres: - image: postgres:16-alpine - container_name: shipsec-postgres - environment: - POSTGRES_USER: shipsec - POSTGRES_PASSWORD: shipsec - POSTGRES_DB: shipsec - POSTGRES_MULTIPLE_DATABASES: temporal - # Internal only - use docker-compose.dev-ports.yml overlay for local dev access - expose: - - "5432" - volumes: - - postgres_data:/var/lib/postgresql/data - - ./init-db:/docker-entrypoint-initdb.d - healthcheck: - test: ["CMD-SHELL", "pg_isready -U shipsec"] - interval: 5s - timeout: 3s - retries: 10 - restart: unless-stopped - - temporal: - image: temporalio/auto-setup:latest - container_name: shipsec-temporal - depends_on: - postgres: - condition: service_healthy - environment: - - DB=postgres12 - - DB_PORT=5432 - - DB_NAME=temporal - - POSTGRES_USER=shipsec - - POSTGRES_PWD=shipsec - - POSTGRES_SEEDS=postgres - - AUTO_SETUP=true - expose: - - "7233" - volumes: - - temporal_data:/var/lib/temporal - restart: unless-stopped - - temporal-ui: - image: temporalio/ui:latest - container_name: shipsec-temporal-ui - depends_on: - - temporal - environment: - - TEMPORAL_ADDRESS=temporal:7233 - - TEMPORAL_CORS_ORIGINS=http://localhost:5173 - expose: - - "8080" - restart: unless-stopped - - minio: - image: minio/minio:RELEASE.2024-10-02T17-50-41Z - container_name: shipsec-minio - command: server /data --console-address ":9001" - environment: - MINIO_ROOT_USER: minioadmin - MINIO_ROOT_PASSWORD: minioadmin - expose: - - "9000" - - "9001" - volumes: - - minio_data:/data - restart: unless-stopped - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] - interval: 30s - timeout: 10s - retries: 5 - - redis: - image: redis:latest - container_name: shipsec-redis - expose: - - "6379" - volumes: - - redis_data:/data - restart: unless-stopped - healthcheck: - test: ["CMD", "redis-cli", "ping"] - interval: 30s - timeout: 10s - retries: 5 - - loki: - image: grafana/loki:3.2.1 - container_name: shipsec-loki - command: -config.file=/etc/loki/local-config.yaml - expose: - - "3100" - volumes: - - ./loki/loki-config.yaml:/etc/loki/local-config.yaml - - loki_data:/loki - restart: unless-stopped - healthcheck: - test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:3100/ready"] - interval: 30s - timeout: 10s - retries: 5 - - redpanda: - image: redpandadata/redpanda:v24.2.5 - container_name: shipsec-redpanda - command: - - redpanda - - start - - --mode=dev-container - - --smp=1 - - --reserve-memory=0M - - --overprovisioned - - --node-id=0 - - --check=false - - --advertise-kafka-addr=localhost:9092 - expose: - - "9092" - - "9644" - volumes: - - redpanda_data:/var/lib/redpanda/data - restart: unless-stopped - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:9644/v1/status/ready"] - interval: 30s - timeout: 10s - retries: 5 - - redpanda-console: - image: redpandadata/console:v2.7.2 - container_name: shipsec-redpanda-console - depends_on: - - redpanda - environment: - CONFIG_FILEPATH: /etc/redpanda/console-config.yaml - expose: - - "8080" - volumes: - - ./redpanda-console-config.yaml:/etc/redpanda/console-config.yaml:ro - restart: unless-stopped - - opensearch: - image: opensearchproject/opensearch:2.11.1 - container_name: shipsec-opensearch - environment: - - discovery.type=single-node - - bootstrap.memory_lock=true - - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" - - DISABLE_SECURITY_PLUGIN=true - - DISABLE_INSTALL_DEMO_CONFIG=true - ulimits: - memlock: - soft: -1 - hard: -1 - nofile: - soft: 65536 - hard: 65536 - # Ports exposed only within Docker network (not to host) - # Use docker-compose.dev-ports.yml overlay for local dev access - expose: - - "9200" - - "9600" - volumes: - - opensearch_data:/usr/share/opensearch/data - restart: unless-stopped - healthcheck: - test: ["CMD-SHELL", "curl -f http://localhost:9200/_cluster/health || exit 1"] - interval: 30s - timeout: 10s - retries: 5 - - opensearch-dashboards: - build: - context: . - dockerfile: opensearch-dashboards.Dockerfile - image: shipsec-opensearch-dashboards:2.11.1 - container_name: shipsec-opensearch-dashboards - depends_on: - opensearch: - condition: service_healthy - environment: - - OPENSEARCH_HOSTS=["http://opensearch:9200"] - - DISABLE_SECURITY_DASHBOARDS_PLUGIN=true - # Ports exposed only within Docker network (not to host) - # Use docker-compose.dev-ports.yml overlay for local dev access - # Production uses nginx reverse proxy at /analytics - expose: - - "5601" - volumes: - - ./opensearch-dashboards.yml:/usr/share/opensearch-dashboards/config/opensearch_dashboards.yml:ro - restart: unless-stopped - healthcheck: - test: ["CMD-SHELL", "curl -f http://localhost:5601/analytics/api/status || exit 1"] - interval: 30s - timeout: 10s - retries: 5 - - # Initialize OpenSearch Dashboards with default index patterns - opensearch-init: - image: curlimages/curl:8.5.0 - container_name: shipsec-opensearch-init - depends_on: - opensearch-dashboards: - condition: service_healthy - volumes: - - ./opensearch-init.sh:/init.sh:ro - entrypoint: ["/bin/sh", "/init.sh"] - restart: "no" - - # Nginx reverse proxy - unified entry point - # DEV MODE: Uses nginx.dev.conf which points to host.docker.internal for PM2 services - nginx: - image: nginx:1.25-alpine - container_name: shipsec-nginx - depends_on: - opensearch-dashboards: - condition: service_healthy - ports: - - "80:80" - volumes: - - ./nginx/nginx.dev.conf:/etc/nginx/nginx.conf:ro - extra_hosts: - - "host.docker.internal:host-gateway" - restart: unless-stopped - healthcheck: - test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost/health"] - interval: 30s - timeout: 10s - retries: 5 - -volumes: - postgres_data: - minio_data: - loki_data: - temporal_data: - redis_data: - redpanda_data: - opensearch_data: - -networks: - default: - name: shipsec-network diff --git a/docker/docker-compose.prod.yml b/docker/docker-compose.prod.yml deleted file mode 100644 index fe81b009b..000000000 --- a/docker/docker-compose.prod.yml +++ /dev/null @@ -1,97 +0,0 @@ -# Production Docker Compose - OpenSearch with Security & Multitenancy -# -# Usage: -# docker compose -f docker-compose.infra.yml -f docker-compose.prod.yml up -d -# -# Prerequisites: -# 1. Generate TLS certificates: ./scripts/generate-certs.sh -# 2. Set environment variables in .env.prod or export them: -# - OPENSEARCH_ADMIN_PASSWORD (required) -# - OPENSEARCH_DASHBOARDS_PASSWORD (required) -# -# This file overrides the development infrastructure with: -# - Security plugin enabled -# - TLS encryption for transport and HTTP -# - Multitenancy support in OpenSearch Dashboards - -services: - opensearch: - # Custom entrypoint for proxy auth config templating - entrypoint: ["/usr/share/opensearch/config/opensearch-security/docker-entrypoint-security.sh"] - environment: - # Remove security disable flags (override dev settings) - - DISABLE_SECURITY_PLUGIN=false - - DISABLE_INSTALL_DEMO_CONFIG=true - # Admin password for healthcheck - - OPENSEARCH_ADMIN_PASSWORD=${OPENSEARCH_ADMIN_PASSWORD:-admin} - # Proxy auth - trusted proxy IP regex (Docker bridge network) - - OPENSEARCH_INTERNAL_PROXIES=172\.\d+\.\d+\.\d+ - # Security configuration - - plugins.security.ssl.transport.pemcert_filepath=certs/node.pem - - plugins.security.ssl.transport.pemkey_filepath=certs/node-key.pem - - plugins.security.ssl.transport.pemtrustedcas_filepath=certs/root-ca.pem - - plugins.security.ssl.transport.enforce_hostname_verification=false - - plugins.security.ssl.http.enabled=true - - plugins.security.ssl.http.pemcert_filepath=certs/node.pem - - plugins.security.ssl.http.pemkey_filepath=certs/node-key.pem - - plugins.security.ssl.http.pemtrustedcas_filepath=certs/root-ca.pem - - plugins.security.allow_unsafe_democertificates=false - - plugins.security.allow_default_init_securityindex=true - - plugins.security.authcz.admin_dn=CN=admin,OU=ShipSec,O=ShipSecAI,L=SF,ST=CA,C=US - - plugins.security.audit.type=internal_opensearch - - plugins.security.enable_snapshot_restore_privilege=true - - plugins.security.check_snapshot_restore_write_privileges=true - - plugins.security.restapi.roles_enabled=["all_access", "security_rest_api_access"] - - cluster.name=shipsec-prod - - node.name=opensearch-node1 - volumes: - - opensearch_data:/usr/share/opensearch/data - - ./certs:/usr/share/opensearch/config/certs:ro - - ./opensearch-security:/usr/share/opensearch/config/opensearch-security:ro - healthcheck: - test: ["CMD-SHELL", "curl -sf -u admin:$${OPENSEARCH_ADMIN_PASSWORD:-admin} --cacert /usr/share/opensearch/config/certs/root-ca.pem https://localhost:9200/_cluster/health || exit 1"] - interval: 30s - timeout: 10s - retries: 10 - start_period: 60s - - opensearch-dashboards: - environment: - # Remove security disable flag (override dev settings) - - DISABLE_SECURITY_DASHBOARDS_PLUGIN=false - - OPENSEARCH_HOSTS=["https://opensearch:9200"] - volumes: - - ./opensearch-dashboards.prod.yml:/usr/share/opensearch-dashboards/config/opensearch_dashboards.yml:ro - - ./certs:/usr/share/opensearch-dashboards/config/certs:ro - healthcheck: - # Check if server responds (401 is fine - means server is up, security just requires auth) - test: ["CMD-SHELL", "curl -sf -o /dev/null -w '%{http_code}' http://localhost:5601/analytics/api/status | grep -qE '(200|401)' || exit 1"] - interval: 30s - timeout: 10s - retries: 10 - start_period: 60s - - # Override init script to work with secured cluster - opensearch-init: - environment: - - OPENSEARCH_SECURITY_ENABLED=true - - OPENSEARCH_CA_CERT=/certs/root-ca.pem - volumes: - - ./opensearch-init.sh:/init.sh:ro - - ./certs:/certs:ro - - # Nginx with production config (container service names) - nginx: - volumes: - - ./nginx/nginx.prod.conf:/etc/nginx/nginx.conf:ro - - ./certs:/etc/nginx/certs:ro - ports: - - "80:80" - - "443:443" - -volumes: - opensearch_data: - -networks: - default: - name: shipsec-network diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml deleted file mode 100644 index 1e907f6ef..000000000 --- a/docker/docker-compose.yml +++ /dev/null @@ -1,42 +0,0 @@ -version: '3.8' - -services: - # AWS Suite - aws-suite: - build: - context: ./mcp-aws-suite - dockerfile: Dockerfile - args: - BASE_IMAGE: shipsec/mcp-stdio-proxy:latest - ports: - - '8081:8080' - environment: - - MCP_COMMAND=${MCP_AWS_COMMAND:-awslabs.cloudtrail-mcp-server} - - MCP_ARGS=${MCP_AWS_ARGS:-[]} - - PORT=8080 - volumes: - - ./mcp-aws-suite:/app - restart: unless-stopped - healthcheck: - test: ['CMD', 'curl', '-f', 'http://localhost:8080/health'] - interval: 30s - timeout: 10s - retries: 3 - start_period: 5s - - # stdio proxy (base service) - mcp-stdio-proxy: - build: - context: ./mcp-stdio-proxy - dockerfile: Dockerfile - ports: - - '8080:8080' - volumes: - - ./mcp-stdio-proxy:/app - restart: unless-stopped - healthcheck: - test: ['CMD', 'curl', '-f', 'http://localhost:8080/health'] - interval: 30s - timeout: 10s - retries: 3 - start_period: 5s diff --git a/docker/init-db/01-create-instance-databases.sh b/docker/init-db/01-create-instance-databases.sh deleted file mode 100755 index 4540fa63a..000000000 --- a/docker/init-db/01-create-instance-databases.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -# Create additional PostgreSQL databases required by ShipSec -# This script is run automatically by PostgreSQL init-entrypoint -# -# Creates: -# - temporal: Required by Temporal workflow engine -# - shipsec_instance_0..9: Multi-instance dev databases - -set -e - -# --- Temporal database (required for workflow engine) --- -echo "🗄️ Creating Temporal database..." -if psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" -d postgres -lqt | cut -d \| -f 1 | grep -qw "temporal"; then - echo " Database temporal already exists, skipping..." -else - psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" -d postgres <<-EOSQL - CREATE DATABASE temporal OWNER "$POSTGRES_USER"; - GRANT ALL PRIVILEGES ON DATABASE temporal TO "$POSTGRES_USER"; -EOSQL - echo " ✅ temporal created" -fi - -# --- Instance-specific databases (for multi-instance dev) --- -echo "🗄️ Creating instance-specific databases..." -for i in {0..9}; do - DB_NAME="shipsec_instance_$i" - - if psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" -d postgres -lqt | cut -d \| -f 1 | grep -qw "$DB_NAME"; then - echo " Database $DB_NAME already exists, skipping..." - else - echo " Creating $DB_NAME..." - psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" -d postgres <<-EOSQL - CREATE DATABASE "$DB_NAME" OWNER "$POSTGRES_USER"; - GRANT ALL PRIVILEGES ON DATABASE "$DB_NAME" TO "$POSTGRES_USER"; -EOSQL - fi -done - -echo "✅ All databases created successfully" diff --git a/docker/loki/loki-config.yaml b/docker/loki/loki-config.yaml deleted file mode 100644 index 788293fc5..000000000 --- a/docker/loki/loki-config.yaml +++ /dev/null @@ -1,51 +0,0 @@ -auth_enabled: false - -server: - http_listen_port: 3100 - grpc_listen_port: 9096 - -common: - path_prefix: /loki - storage: - filesystem: - chunks_directory: /loki/chunks - rules_directory: /loki/rules - replication_factor: 1 - ring: - instance_addr: 127.0.0.1 - kvstore: - store: inmemory - -limits_config: - allow_structured_metadata: false - -query_range: - results_cache: - cache: - embedded_cache: - enabled: true - max_size_mb: 100 - -schema_config: - configs: - - from: 2020-10-24 - store: boltdb-shipper - object_store: filesystem - schema: v11 - index: - prefix: index_ - period: 24h - -ruler: - alertmanager_url: http://localhost:9093 - -# By default, Loki will send anonymous, but uniquely-identifiable usage and configuration -# analytics to Grafana Labs. These statistics are sent to https://stats.grafana.org/ -# -# Statistics help us understand how Loki is used, and they show us performance -# levels for most users. This helps us prioritize features and documentation. -# For more information on what's sent, look at -# https://github.com/grafana/loki/blob/main/pkg/usagestats/stats.go -# To turn off analytics, uncomment the following: -# analytics: -# reporting_enabled: false \ No newline at end of file diff --git a/docker/mcp-aws-cloudtrail/Dockerfile b/docker/mcp-aws-cloudtrail/Dockerfile deleted file mode 100644 index 43a0ccffa..000000000 --- a/docker/mcp-aws-cloudtrail/Dockerfile +++ /dev/null @@ -1,11 +0,0 @@ -ARG BASE_IMAGE=shipsec/mcp-stdio-proxy:latest -FROM ${BASE_IMAGE} - -RUN apt-get update \ - && apt-get install -y --no-install-recommends python3 python3-pip \ - && rm -rf /var/lib/apt/lists/* \ - && pip install --no-cache-dir --break-system-packages uv \ - && uv pip install --system --break-system-packages awslabs-cloudtrail-mcp-server - -ENV MCP_COMMAND=awslabs.cloudtrail-mcp-server -ENV MCP_ARGS='[]' diff --git a/docker/mcp-aws-cloudtrail/README.md b/docker/mcp-aws-cloudtrail/README.md deleted file mode 100644 index 7828a5d6b..000000000 --- a/docker/mcp-aws-cloudtrail/README.md +++ /dev/null @@ -1,22 +0,0 @@ -# AWS CloudTrail MCP Proxy Image - -This image extends the MCP stdio proxy and installs the CloudTrail MCP server. - -## Build - -```bash -docker build -t shipsec/mcp-aws-cloudtrail:latest docker/mcp-aws-cloudtrail -``` - -## Run (example) - -```bash -docker run --rm -p 8080:8080 \ - -e AWS_ACCESS_KEY_ID=... \ - -e AWS_SECRET_ACCESS_KEY=... \ - -e AWS_SESSION_TOKEN=... \ - -e AWS_REGION=us-east-1 \ - shipsec/mcp-aws-cloudtrail:latest -``` - -The proxy exposes MCP on `http://localhost:8080/mcp`. diff --git a/docker/mcp-aws-cloudwatch/Dockerfile b/docker/mcp-aws-cloudwatch/Dockerfile deleted file mode 100644 index 4283d8c5c..000000000 --- a/docker/mcp-aws-cloudwatch/Dockerfile +++ /dev/null @@ -1,14 +0,0 @@ -ARG BASE_IMAGE=shipsec/mcp-stdio-proxy:latest -FROM ${BASE_IMAGE} - -RUN apt-get update \ - && apt-get install -y --no-install-recommends \ - python3 python3-pip python3-dev \ - build-essential gfortran \ - libopenblas-dev liblapack-dev \ - && rm -rf /var/lib/apt/lists/* \ - && pip install --no-cache-dir --break-system-packages uv \ - && uv pip install --system --break-system-packages awslabs-cloudwatch-mcp-server - -ENV MCP_COMMAND=awslabs.cloudwatch-mcp-server -ENV MCP_ARGS='[]' diff --git a/docker/mcp-aws-cloudwatch/README.md b/docker/mcp-aws-cloudwatch/README.md deleted file mode 100644 index 4b8f57d02..000000000 --- a/docker/mcp-aws-cloudwatch/README.md +++ /dev/null @@ -1,22 +0,0 @@ -# AWS CloudWatch MCP Proxy Image - -This image extends the MCP stdio proxy and installs the CloudWatch MCP server. - -## Build - -```bash -docker build -t shipsec/mcp-aws-cloudwatch:latest docker/mcp-aws-cloudwatch -``` - -## Run (example) - -```bash -docker run --rm -p 8080:8080 \ - -e AWS_ACCESS_KEY_ID=... \ - -e AWS_SECRET_ACCESS_KEY=... \ - -e AWS_SESSION_TOKEN=... \ - -e AWS_REGION=us-east-1 \ - shipsec/mcp-aws-cloudwatch:latest -``` - -The proxy exposes MCP on `http://localhost:8080/mcp`. diff --git a/docker/mcp-aws-suite/Dockerfile b/docker/mcp-aws-suite/Dockerfile deleted file mode 100644 index 5a49227b8..000000000 --- a/docker/mcp-aws-suite/Dockerfile +++ /dev/null @@ -1,64 +0,0 @@ -ARG BASE_IMAGE=shipsec/mcp-stdio-proxy:latest -FROM ${BASE_IMAGE} - -# Install Python and uv for AWS MCP servers -RUN apt-get update \ - && apt-get install -y --no-install-recommends python3 python3-pip python3-dev build-essential \ - && rm -rf /var/lib/apt/lists/* \ - && pip install --no-cache-dir --break-system-packages uv - -# Install 10 essential AWS security MCP servers -# Core security & auditing -RUN uv pip install --system --break-system-packages awslabs.cloudtrail-mcp-server -RUN uv pip install --system --break-system-packages awslabs.iam-mcp-server -RUN uv pip install --system --break-system-packages awslabs.s3-tables-mcp-server -RUN uv pip install --system --break-system-packages awslabs.cloudwatch-mcp-server - -# Networking & infrastructure security -RUN uv pip install --system --break-system-packages awslabs.aws-network-mcp-server - -# Serverless security -RUN uv pip install --system --break-system-packages awslabs.lambda-tool-mcp-server - -# Database security -RUN uv pip install --system --break-system-packages awslabs.dynamodb-mcp-server - -# Documentation & best practices -RUN uv pip install --system --break-system-packages awslabs.aws-documentation-mcp-server -RUN uv pip install --system --break-system-packages awslabs.well-architected-security-mcp-server - -# API explorer (for any AWS service) -RUN uv pip install --system --break-system-packages awslabs.aws-api-mcp-server - -# Compatibility shim for fastmcp expecting streamable_http_client -RUN python3 - <<'PY' -from pathlib import Path - -content = """ -try: - import mcp.client.streamable_http as _m - if not hasattr(_m, 'streamable_http_client') and hasattr(_m, 'streamablehttp_client'): - _m.streamable_http_client = _m.streamablehttp_client -except Exception: - pass -""" - -# Prefer usercustomize (loaded after sitecustomize) to avoid clobbering system hooks. -usercustomize = Path('/usr/local/lib/python3.11/dist-packages/usercustomize.py') -usercustomize.write_text(content) - -# Also append to system sitecustomize if present to ensure patch executes. -sitecustomize = Path('/usr/lib/python3/dist-packages/sitecustomize.py') -if sitecustomize.exists(): - existing = sitecustomize.read_text() - if 'streamable_http_client' not in existing: - sitecustomize.write_text(existing + "\\n" + content) -PY - -# Set default MCP command (can be overridden via MCP_COMMAND env var) -ENV MCP_COMMAND=awslabs.cloudtrail-mcp-server -ENV MCP_ARGS='[]' - -# Health check -HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \ - CMD curl -f http://localhost:8080/health || exit 1 diff --git a/docker/mcp-aws-suite/README.md b/docker/mcp-aws-suite/README.md deleted file mode 100644 index 2dc3a01cc..000000000 --- a/docker/mcp-aws-suite/README.md +++ /dev/null @@ -1,66 +0,0 @@ -# AWS Suite MCP Docker Image - -This Docker image contains multiple AWS-related MCP (Model Context Protocol) servers bundled together for easy deployment and testing. - -## Included MCP Servers - -- **awslabs.cloudtrail-mcp-server** - AWS CloudTrail integration -- **awslabs.cloudwatch-mcp-server** - AWS CloudWatch integration -- **awslabs.ec2-mcp-server** - AWS EC2 integration -- **awslabs.s3-mcp-server** - AWS S3 integration - -## Usage - -### Building the Image - -```bash -docker build -t shipsec/mcp-aws-suite:latest . -``` - -### Running the Container - -With default settings (CloudTrail): - -```bash -docker run -p 8080:8080 shipsec/mcp-aws-suite:latest -``` - -With different MCP server: - -```bash -docker run -e MCP_COMMAND=awslabs.cloudwatch-mcp-server -p 8080:8080 shipsec/mcp-aws-suite:latest -``` - -With custom arguments: - -```bash -docker run -e MCP_COMMAND=awslabs.cloudtrail-mcp-server -e MCP_ARGS='["--region", "us-west-2"]' -p 8080:8080 shipsec/mcp-aws-suite:latest -``` - -### Environment Variables - -- `MCP_COMMAND` (required): The MCP server to run. Defaults to `awslabs.cloudtrail-mcp-server` -- `MCP_ARGS` (optional): JSON array of command arguments -- `PORT`: HTTP port for the stdio proxy (defaults to 8080) - -### Health Check - -The container exposes a health check endpoint at `/health`: - -```bash -curl http://localhost:8080/health -``` - -### AWS Credentials - -To use the AWS MCP servers, you'll need to provide AWS credentials. You can mount them: - -```bash -docker run -v ~/.aws/credentials:/root/.aws/credentials:ro -e AWS_PROFILE=default -p 8080:8080 shipsec/mcp-aws-suite:latest -``` - -or set environment variables: - -```bash -docker run -e AWS_ACCESS_KEY_ID=your_access_key -e AWS_SECRET_ACCESS_KEY=your_secret_key -p 8080:8080 shipsec/mcp-aws-suite:latest -``` diff --git a/docker/mcp-aws-suite/named-servers.json b/docker/mcp-aws-suite/named-servers.json deleted file mode 100644 index f2da63a51..000000000 --- a/docker/mcp-aws-suite/named-servers.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "mcpServers": { - "cloudtrail": { - "command": "awslabs.cloudtrail-mcp-server", - "args": [] - }, - "iam": { - "command": "awslabs.iam-mcp-server", - "args": [] - }, - "lambda": { - "command": "awslabs.lambda-tool-mcp-server", - "args": [] - } - } -} diff --git a/docker/mcp-stdio-proxy/Dockerfile b/docker/mcp-stdio-proxy/Dockerfile deleted file mode 100644 index 7ab90b04f..000000000 --- a/docker/mcp-stdio-proxy/Dockerfile +++ /dev/null @@ -1,14 +0,0 @@ -FROM node:24-slim - -WORKDIR /app - -COPY package.json ./ -RUN npm install --omit=dev - -COPY server.mjs ./ -COPY named-servers.json ./ - -ENV PORT=8080 -EXPOSE 8080 - -CMD ["node", "server.mjs"] diff --git a/docker/mcp-stdio-proxy/README.md b/docker/mcp-stdio-proxy/README.md deleted file mode 100644 index 9785829bd..000000000 --- a/docker/mcp-stdio-proxy/README.md +++ /dev/null @@ -1,31 +0,0 @@ -# MCP Stdio Proxy - -This image wraps a stdio-based MCP server and exposes it over Streamable HTTP. - -## Build - -```bash -docker build -t shipsec/mcp-stdio-proxy:latest docker/mcp-stdio-proxy -``` - -## Run - -```bash -docker run --rm -p 8080:8080 \ - -e MCP_COMMAND=uvx \ - -e MCP_ARGS='["awslabs-cloudwatch-mcp-server"]' \ - shipsec/mcp-stdio-proxy:latest -``` - -The proxy will expose MCP on `http://localhost:8080/mcp` and a basic health endpoint at `/health`. - -## Environment - -- `MCP_COMMAND` (required): Command to launch the stdio MCP server. -- `MCP_ARGS` (optional): JSON array or space-delimited list of arguments. -- `PORT` / `MCP_PORT` (optional): Port for the HTTP server (default: 8080). - -## Notes - -- The proxy lists tools once at startup and registers them. Restart the container if tools change. -- Make sure the stdio server binary is present in the image. For third-party tools, build a derived image that installs them. diff --git a/docker/mcp-stdio-proxy/named-servers.json b/docker/mcp-stdio-proxy/named-servers.json deleted file mode 100644 index da39e4ffa..000000000 --- a/docker/mcp-stdio-proxy/named-servers.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "mcpServers": {} -} diff --git a/docker/mcp-stdio-proxy/package.json b/docker/mcp-stdio-proxy/package.json deleted file mode 100644 index e766511d6..000000000 --- a/docker/mcp-stdio-proxy/package.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "name": "mcp-stdio-proxy", - "version": "0.1.0", - "private": true, - "type": "module", - "description": "HTTP proxy for stdio-based MCP servers", - "scripts": { - "start": "node server.mjs" - }, - "dependencies": { - "@modelcontextprotocol/sdk": "^1.25.3", - "express": "^5.2.1" - } -} diff --git a/docker/mcp-stdio-proxy/server.mjs b/docker/mcp-stdio-proxy/server.mjs deleted file mode 100644 index 4f12d1ed0..000000000 --- a/docker/mcp-stdio-proxy/server.mjs +++ /dev/null @@ -1,298 +0,0 @@ -import express from 'express'; -import { Client } from '@modelcontextprotocol/sdk/client/index.js'; -import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js'; -import { - LATEST_PROTOCOL_VERSION, -} from '@modelcontextprotocol/sdk/types.js'; -import { readFileSync } from 'fs'; -import { fileURLToPath } from 'url'; -import { dirname, join } from 'path'; - -const __filename = fileURLToPath(import.meta.url); -const __dirname = dirname(__filename); - -function parseArgs(raw) { - if (!raw) return []; - try { - const parsed = JSON.parse(raw); - if (Array.isArray(parsed)) return parsed.map(String); - } catch { - // fall through - } - return raw - .split(' ') - .map((entry) => entry.trim()) - .filter(Boolean); -} - -// Parse named servers config from JSON file or MCP_NAMED_SERVERS env var -function parseNamedServersConfig() { - // Try env var first (JSON string) - if (process.env.MCP_NAMED_SERVERS) { - try { - return JSON.parse(process.env.MCP_NAMED_SERVERS); - } catch (err) { - console.error('[mcp-proxy] Failed to parse MCP_NAMED_SERVERS JSON:', err.message); - } - } - - // Try config file path - if (process.env.MCP_NAMED_SERVERS_CONFIG) { - try { - const configPath = process.env.MCP_NAMED_SERVERS_CONFIG; - const configContent = readFileSync(configPath, 'utf-8'); - return JSON.parse(configContent); - } catch (err) { - console.error('[mcp-proxy] Failed to read MCP_NAMED_SERVERS_CONFIG file:', err.message); - } - } - - // Try default config file location - const defaultConfigPath = join(__dirname, 'named-servers.json'); - try { - const configContent = readFileSync(defaultConfigPath, 'utf-8'); - return JSON.parse(configContent); - } catch (err) { - // Config file doesn't exist, not an error - } - - return null; -} - -/** - * Handle a JSON-RPC request by forwarding to the stdio MCP client. - * - * This bypasses the MCP SDK's Server class which only accepts one `initialize` - * per lifetime. By handling JSON-RPC directly, we support unlimited HTTP clients - * (e.g. worker for discovery, then gateway for tool calls) sharing one stdio server. - */ -async function handleJsonRpc(req, res, stdioClient, name) { - const body = req.body; - - // Notifications have no `id` — return 202 Accepted (expected by MCP SDK client) - if (body && body.method && body.id === undefined) { - return res.status(202).end(); - } - - if (!body || !body.method) { - return res.status(400).json({ - jsonrpc: '2.0', - id: body?.id ?? null, - error: { code: -32600, message: 'Invalid request: missing method' }, - }); - } - - try { - switch (body.method) { - case 'initialize': { - const result = { - protocolVersion: LATEST_PROTOCOL_VERSION, - capabilities: stdioClient.getServerCapabilities() ?? { tools: { listChanged: false } }, - serverInfo: stdioClient.getServerVersion() ?? { - name: `mcp-proxy-${name}`, - version: '1.0.0', - }, - instructions: stdioClient.getInstructions?.(), - }; - return res.json({ jsonrpc: '2.0', id: body.id, result }); - } - - case 'tools/list': { - const result = await stdioClient.listTools(); - return res.json({ jsonrpc: '2.0', id: body.id, result }); - } - - case 'tools/call': { - const result = await stdioClient.callTool({ - name: body.params.name, - arguments: body.params.arguments ?? {}, - }); - return res.json({ jsonrpc: '2.0', id: body.id, result }); - } - - case 'resources/list': { - const result = await stdioClient.listResources(); - return res.json({ jsonrpc: '2.0', id: body.id, result }); - } - - case 'resources/read': { - const result = await stdioClient.readResource({ uri: body.params.uri }); - return res.json({ jsonrpc: '2.0', id: body.id, result }); - } - - case 'prompts/list': { - const result = await stdioClient.listPrompts(); - return res.json({ jsonrpc: '2.0', id: body.id, result }); - } - - case 'prompts/get': { - const result = await stdioClient.getPrompt({ - name: body.params.name, - arguments: body.params.arguments ?? {}, - }); - return res.json({ jsonrpc: '2.0', id: body.id, result }); - } - - default: - return res.status(400).json({ - jsonrpc: '2.0', - id: body.id, - error: { code: -32601, message: `Method not found: ${body.method}` }, - }); - } - } catch (error) { - console.error(`[mcp-proxy] Error handling ${body.method} for '${name}':`, error.message); - return res.status(200).json({ - jsonrpc: '2.0', - id: body.id, - error: { code: -32603, message: error.message }, - }); - } -} - -const port = Number.parseInt(process.env.PORT || process.env.MCP_PORT || '8080', 10); - -// Check if we have named servers configuration -const namedServersConfig = parseNamedServersConfig(); -const hasNamedServers = namedServersConfig && namedServersConfig.mcpServers; - -// Legacy mode: single server via MCP_COMMAND -const command = process.env.MCP_COMMAND; -const args = parseArgs(process.env.MCP_ARGS || ''); - -// Map to store connected stdio clients for named servers -// name -> { client } -const namedClients = new Map(); - -if (hasNamedServers) { - console.log('[mcp-proxy] Starting in NAMED SERVERS mode'); - - // Initialize all named servers (stdio connections only) - for (const [name, serverConfig] of Object.entries(namedServersConfig.mcpServers)) { - try { - console.log(`[mcp-proxy] Initializing named server: ${name}`); - console.log(`[mcp-proxy] command: ${serverConfig.command}`); - console.log(`[mcp-proxy] args: ${serverConfig.args?.join(' ') || '(none)'}`); - - const client = new Client({ - name: `mcp-proxy-${name}`, - version: '1.0.0' - }); - - const clientTransport = new StdioClientTransport({ - command: serverConfig.command, - args: serverConfig.args || [], - env: serverConfig.env || {}, - }); - - await client.connect(clientTransport); - - namedClients.set(name, { client }); - console.log(`[mcp-proxy] Named server '${name}' ready`); - } catch (err) { - console.error(`[mcp-proxy] Failed to initialize named server '${name}':`, err.message); - } - } - - console.log(`[mcp-proxy] Initialized ${namedClients.size} named server(s)`); -} else { - // Legacy single-server mode - console.log('[mcp-proxy] Starting in SINGLE SERVER mode (legacy)'); - - if (!command) { - console.error('MCP_COMMAND is required to start the stdio MCP server in single-server mode.'); - process.exit(1); - } - - const client = new Client({ name: 'shipsec-mcp-stdio-proxy', version: '1.0.0' }); - const clientTransport = new StdioClientTransport({ - command, - args, - }); - - await client.connect(clientTransport); - - namedClients.set('__default__', { client }); - console.log(`[mcp-proxy] Single server mode ready: ${command} ${args.join(' ')}`); -} - -const app = express(); -app.use(express.json({ limit: '2mb' })); - -// Health check endpoint -app.get('/health', (_req, res) => { - const serverNames = hasNamedServers - ? Object.keys(namedServersConfig.mcpServers) - : ['__default__']; - - res.json({ - status: 'ok', - mode: hasNamedServers ? 'named-servers' : 'single-server', - servers: serverNames.map(name => ({ - name: name === '__default__' ? 'default' : name, - ready: namedClients.has(name), - })), - }); -}); - -// List available named servers -app.get('/servers', (_req, res) => { - if (!hasNamedServers) { - return res.json({ servers: [{ name: 'default', path: '/mcp' }] }); - } - - res.json({ - servers: Object.keys(namedServersConfig.mcpServers).map(name => ({ - name, - path: `/servers/${name}/sse`, - })), - }); -}); - -// Legacy endpoint for single-server mode — POST handles JSON-RPC, GET/DELETE return 405 -app.post('/mcp', async (req, res) => { - const namedClient = namedClients.get('__default__'); - if (!namedClient) { - return res.status(503).json({ error: 'No MCP server connected' }); - } - - await handleJsonRpc(req, res, namedClient.client, 'default'); -}); - -app.get('/mcp', (_req, res) => res.status(405).json({ error: 'SSE not supported, use POST' })); -app.delete('/mcp', (_req, res) => res.status(405).json({ error: 'Session cleanup not needed' })); - -// Named server endpoints: /servers/:name/sse -app.post('/servers/:name/sse', async (req, res) => { - const { name } = req.params; - const namedClient = namedClients.get(name); - - if (!namedClient) { - console.error(`[mcp-proxy] Unknown named server: ${name}`); - return res.status(404).json({ - error: `Named server '${name}' not found`, - availableServers: Array.from(namedClients.keys()), - }); - } - - await handleJsonRpc(req, res, namedClient.client, name); -}); - -app.get('/servers/:name/sse', (_req, res) => - res.status(405).json({ error: 'SSE not supported, use POST' }) -); -app.delete('/servers/:name/sse', (_req, res) => - res.status(405).json({ error: 'Session cleanup not needed' }) -); - -app.listen(port, '0.0.0.0', () => { - console.log(`[mcp-proxy] Listening on http://0.0.0.0:${port}`); - if (hasNamedServers) { - console.log(`[mcp-proxy] Named servers mode:`); - for (const name of Object.keys(namedServersConfig.mcpServers)) { - console.log(`[mcp-proxy] - /servers/${name}/sse`); - } - } else { - console.log(`[mcp-proxy] Single server mode: /mcp`); - } -}); diff --git a/docker/nginx/nginx.dev.conf b/docker/nginx/nginx.dev.conf deleted file mode 100644 index c148d5491..000000000 --- a/docker/nginx/nginx.dev.conf +++ /dev/null @@ -1,242 +0,0 @@ -worker_processes auto; -error_log /var/log/nginx/error.log warn; -pid /var/run/nginx.pid; - -events { - worker_connections 1024; -} - -http { - include /etc/nginx/mime.types; - default_type application/octet-stream; - - log_format main '$remote_addr - $remote_user [$time_local] "$request" ' - '$status $body_bytes_sent "$http_referer" ' - '"$http_user_agent" "$http_x_forwarded_for"'; - - # Debug log format for analytics auth issues - log_format auth_debug '$remote_addr [$time_local] "$request" $status ' - 'auth_org_id="$auth_org_id" auth_user="$auth_user_id"'; - - access_log /var/log/nginx/access.log main; - - sendfile on; - tcp_nopush on; - tcp_nodelay on; - keepalive_timeout 65; - - # Gzip compression - gzip on; - gzip_vary on; - gzip_proxied any; - gzip_comp_level 6; - gzip_types text/plain text/css text/xml application/json application/javascript - application/rss+xml application/atom+xml image/svg+xml; - - # ================================================================= - # DEVELOPMENT MODE - Frontend & Backend run on host via PM2 - # Uses host.docker.internal to reach host machine from container - # - # NOTE: These upstreams hardcode instance 0 ports (5173/3211). - # Nginx is shared infra and always proxies to instance 0. - # Multi-instance users (SHIPSEC_INSTANCE=N where N>0) should - # access their instance directly via its ports: - # Frontend: http://localhost:<5173 + N*100> - # Backend: http://localhost:<3211 + N*100> - # See docs/MULTI-INSTANCE-DEV.md for details. - # ================================================================= - - # Upstream definitions - pointing to host machine (PM2 services, instance 0) - upstream frontend { - # Vite dev server on host (instance 0) - server host.docker.internal:5173; - keepalive 32; - } - - upstream backend { - # NestJS backend on host (instance 0) - server host.docker.internal:3211; - keepalive 32; - } - - # OpenSearch Dashboards runs in Docker - upstream opensearch-dashboards { - server opensearch-dashboards:5601; - keepalive 32; - } - - # WebSocket connection upgrade map - # IMPORTANT: Use '' (empty) not 'close' for non-WebSocket requests. - # 'close' kills upstream keepalive, forcing a new TCP connection per request - # through Docker's networking stack — adding 10ms-3s latency per request. - map $http_upgrade $connection_upgrade { - default upgrade; - '' ''; - } - - server { - listen 80; - server_name _; - - # Client request body size (for file uploads) - client_max_body_size 100M; - client_body_buffer_size 10M; - - # Proxy buffer settings - proxy_buffer_size 128k; - proxy_buffers 4 256k; - proxy_busy_buffers_size 256k; - - # Common proxy headers - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header X-Forwarded-Host $host; - - # ================================================================= - # Auth validation endpoint (public, proxied to backend) - # ================================================================= - location = /auth/validate { - proxy_pass http://backend/api/v1/auth/validate; - proxy_set_header Cookie $http_cookie; - proxy_set_header Authorization $http_authorization; - } - - # ================================================================= - # Internal auth validation endpoint for auth_request - # ================================================================= - location = /_auth { - internal; - # Debug: log internal auth requests - access_log /var/log/nginx/auth_internal.log main; - - proxy_pass http://backend/api/v1/auth/validate; - proxy_pass_request_body off; - proxy_set_header Content-Length ""; - proxy_set_header X-Original-URI $request_uri; - # Pass cookies for session auth - proxy_set_header Cookie $http_cookie; - # Pass Authorization header for API key/token auth - proxy_set_header Authorization $http_authorization; - } - - # ================================================================= - # Dashboards SaaS Lockdown - Whitelist allowed app pages - # Unwanted plugins are removed from the image (opensearch-dashboards.Dockerfile) - # This regex is defense-in-depth against any remaining/future plugins - # Admin: use direct Dashboards port (5601) bypassing nginx - # ================================================================= - location ~ ^/analytics/app/(?!(discover|visualize|dashboards?|dev_tools|alerting|notifications|management|data-explorer|home)($|/|\?|#)) { - return 403; - } - - # ================================================================= - # OpenSearch Dashboards - /analytics/* (PROTECTED + TENANT ISOLATED) - # ================================================================= - location /analytics/ { - # Debug logging for auth issues - access_log /var/log/nginx/analytics_auth.log auth_debug; - - # Require authentication before proxying - auth_request /_auth; - # On auth failure, redirect to login page - error_page 401 = @auth_redirect; - - # Capture org/user context from auth response headers - auth_request_set $auth_org_id $upstream_http_x_auth_organization_id; - auth_request_set $auth_user_id $upstream_http_x_auth_user_id; - - # NOTE: Cannot use 'if ($auth_org_id = "")' here because nginx's 'if' runs - # in the rewrite phase BEFORE auth_request completes in the access phase. - # OpenSearch Security will reject requests with invalid/missing proxy auth headers. - # For fail-closed behavior, the auth backend should return 401 if org context is missing. - - proxy_pass http://opensearch-dashboards; - - # Standard forwarding headers (must be repeated here because nginx - # proxy_set_header in a location block overrides ALL parent-level directives) - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - - # WebSocket support for dashboards real-time features - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection $connection_upgrade; - - # Timeouts for dashboards (can be slow for large queries) - proxy_connect_timeout 60s; - proxy_send_timeout 120s; - proxy_read_timeout 120s; - - # Dashboards-specific headers - proxy_set_header osd-xsrf "true"; - - # OpenSearch Security proxy auth headers - # proxy_set_header REPLACES any client-supplied headers (prevents spoofing) - proxy_set_header x-proxy-user $auth_org_id; - proxy_set_header x-proxy-roles "customer_${auth_org_id}_ro"; - proxy_set_header securitytenant $auth_org_id; - - # Preserve cookies - proxy_cookie_path /analytics/ /analytics/; - - # No redirect rewriting needed - we preserve the path - proxy_redirect off; - } - - # Auth redirect handler - redirect to home with return URL - location @auth_redirect { - return 302 /?returnTo=$request_uri; - } - - # Exact match for /analytics without trailing slash - location = /analytics { - return 301 /analytics/; - } - - # ================================================================= - # Backend API - /api/* - # ================================================================= - location /api/ { - proxy_pass http://backend/api/; - - # WebSocket support for terminal/streaming endpoints - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection $connection_upgrade; - - # API timeouts - proxy_connect_timeout 30s; - proxy_send_timeout 60s; - proxy_read_timeout 60s; - - # Don't buffer API responses (important for streaming) - proxy_buffering off; - } - - # ================================================================= - # Frontend (SPA) - /* (catch-all) - # ================================================================= - location / { - proxy_pass http://frontend/; - - # WebSocket support for Vite HMR in development - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection $connection_upgrade; - - # Frontend timeouts - longer read timeout for HMR WebSocket - proxy_connect_timeout 30s; - proxy_send_timeout 30s; - proxy_read_timeout 86400s; # 24 hours - keep HMR WebSocket alive - } - - # Health check endpoint - location /health { - access_log off; - return 200 "healthy\n"; - add_header Content-Type text/plain; - } - } -} diff --git a/docker/nginx/nginx.prod.conf b/docker/nginx/nginx.prod.conf deleted file mode 100644 index 3677280a6..000000000 --- a/docker/nginx/nginx.prod.conf +++ /dev/null @@ -1,208 +0,0 @@ -worker_processes auto; -error_log /var/log/nginx/error.log warn; -pid /var/run/nginx.pid; - -events { - worker_connections 1024; -} - -http { - include /etc/nginx/mime.types; - default_type application/octet-stream; - - log_format main '$remote_addr - $remote_user [$time_local] "$request" ' - '$status $body_bytes_sent "$http_referer" ' - '"$http_user_agent" "$http_x_forwarded_for"'; - - access_log /var/log/nginx/access.log main; - - sendfile on; - tcp_nopush on; - tcp_nodelay on; - keepalive_timeout 65; - - # Gzip compression - gzip on; - gzip_vary on; - gzip_proxied any; - gzip_comp_level 6; - gzip_types text/plain text/css text/xml application/json application/javascript - application/rss+xml application/atom+xml image/svg+xml; - - # Upstream definitions - upstream frontend { - server frontend:8080; - keepalive 32; - } - - upstream backend { - server backend:3211; - keepalive 32; - } - - upstream opensearch-dashboards { - server opensearch-dashboards:5601; - keepalive 32; - } - - # WebSocket connection upgrade map - map $http_upgrade $connection_upgrade { - default upgrade; - '' close; - } - - server { - listen 80; - server_name _; - - # Client request body size (for file uploads) - client_max_body_size 100M; - client_body_buffer_size 10M; - - # Proxy buffer settings - proxy_buffer_size 128k; - proxy_buffers 4 256k; - proxy_busy_buffers_size 256k; - - # Common proxy headers - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header X-Forwarded-Host $host; - - # ================================================================= - # Auth validation endpoint (public, proxied to backend) - # ================================================================= - location = /auth/validate { - proxy_pass http://backend/api/v1/auth/validate; - proxy_set_header Cookie $http_cookie; - proxy_set_header Authorization $http_authorization; - } - - # ================================================================= - # Internal auth validation endpoint for auth_request - # ================================================================= - location = /_auth { - internal; - proxy_pass http://backend/api/v1/auth/validate; - proxy_pass_request_body off; - proxy_set_header Content-Length ""; - proxy_set_header X-Original-URI $request_uri; - # Pass cookies for session auth - proxy_set_header Cookie $http_cookie; - # Pass Authorization header for API key/token auth - proxy_set_header Authorization $http_authorization; - } - - # ================================================================= - # Dashboards SaaS Lockdown - Whitelist allowed app pages - # Unwanted plugins are removed from the image (opensearch-dashboards.Dockerfile) - # This regex is defense-in-depth against any remaining/future plugins - # Admin: use direct Dashboards port (5601) bypassing nginx - # ================================================================= - location ~ ^/analytics/app/(?!(discover|visualize|dashboards?|dev_tools|alerting|notifications|management|data-explorer|home)($|/|\?|#)) { - return 403; - } - - # ================================================================= - # OpenSearch Dashboards - /analytics/* (PROTECTED + TENANT ISOLATED) - # ================================================================= - location /analytics/ { - # Require authentication before proxying - auth_request /_auth; - # On auth failure, redirect to login page - error_page 401 = @auth_redirect; - - # Capture org/user context from auth response headers - auth_request_set $auth_org_id $upstream_http_x_auth_organization_id; - auth_request_set $auth_user_id $upstream_http_x_auth_user_id; - - # FAIL-CLOSED: Reject if org context is missing - # This prevents unauthenticated or org-less sessions from reaching Dashboards - if ($auth_org_id = "") { - return 403; - } - - proxy_pass http://opensearch-dashboards/; - - # WebSocket support for dashboards real-time features - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection $connection_upgrade; - - # Timeouts for dashboards (can be slow for large queries) - proxy_connect_timeout 60s; - proxy_send_timeout 120s; - proxy_read_timeout 120s; - - # Dashboards-specific headers - proxy_set_header osd-xsrf "true"; - - # OpenSearch Security proxy auth headers - # proxy_set_header REPLACES any client-supplied headers (prevents spoofing) - proxy_set_header x-proxy-user $auth_org_id; - proxy_set_header x-proxy-roles "customer_${auth_org_id}_ro"; - proxy_set_header securitytenant $auth_org_id; - - # Preserve cookies - proxy_cookie_path / /analytics/; - - # Handle redirects from dashboards - proxy_redirect / /analytics/; - proxy_redirect http://opensearch-dashboards:5601/ /analytics/; - } - - # Auth redirect handler - redirect to home with return URL - location @auth_redirect { - return 302 /?returnTo=$request_uri; - } - - # Exact match for /analytics without trailing slash - location = /analytics { - return 301 /analytics/; - } - - # ================================================================= - # Backend API - /api/* - # ================================================================= - location /api/ { - proxy_pass http://backend/api/; - - # WebSocket support for terminal/streaming endpoints - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection $connection_upgrade; - - # API timeouts - proxy_connect_timeout 30s; - proxy_send_timeout 60s; - proxy_read_timeout 60s; - - # Don't buffer API responses (important for streaming) - proxy_buffering off; - } - - # ================================================================= - # Frontend (SPA) - /* (catch-all) - # ================================================================= - location / { - proxy_pass http://frontend/; - - # WebSocket support for Vite HMR in development - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection $connection_upgrade; - - # Frontend timeouts - proxy_connect_timeout 30s; - proxy_send_timeout 30s; - proxy_read_timeout 30s; - } - - # Health check endpoint - location /health { - access_log off; - return 200 "healthy\n"; - add_header Content-Type text/plain; - } - } -} diff --git a/docker/opensearch-dashboards.Dockerfile b/docker/opensearch-dashboards.Dockerfile deleted file mode 100644 index 65613ffb5..000000000 --- a/docker/opensearch-dashboards.Dockerfile +++ /dev/null @@ -1,18 +0,0 @@ -# Custom OpenSearch Dashboards image for SaaS tenant lockdown -# Source: https://github.com/ShipSecAI/tools/tree/main/misc/opensearch-dashboards-saas -# -# Removes unwanted plugins from sidebar. Config-level disabling is NOT possible -# because OSD 2.x plugins don't register an "enabled" config key (fatal error). -# See the tools repo README for full documentation. - -FROM opensearchproject/opensearch-dashboards:2.11.1 - -RUN /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin remove queryWorkbenchDashboards && \ - /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin remove reportsDashboards && \ - /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin remove anomalyDetectionDashboards && \ - /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin remove customImportMapDashboards && \ - /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin remove securityAnalyticsDashboards && \ - /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin remove searchRelevanceDashboards && \ - /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin remove mlCommonsDashboards && \ - /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin remove indexManagementDashboards && \ - /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin remove observabilityDashboards diff --git a/docker/opensearch-dashboards.prod.yml b/docker/opensearch-dashboards.prod.yml deleted file mode 100644 index c9007b136..000000000 --- a/docker/opensearch-dashboards.prod.yml +++ /dev/null @@ -1,66 +0,0 @@ -# OpenSearch Dashboards Production Configuration -# Mount this file to /usr/share/opensearch-dashboards/config/opensearch_dashboards.yml -# -# This configuration enables: -# - Security plugin with authentication -# - Multitenancy for tenant isolation -# - TLS for secure communication with OpenSearch - -server.host: "0.0.0.0" -server.port: 5601 - -# Base path configuration for reverse proxy -server.basePath: "/analytics" -server.rewriteBasePath: true - -# OpenSearch connection (HTTPS for production) -opensearch.hosts: ["https://opensearch:9200"] - -# TLS Configuration - trust the CA certificate -opensearch.ssl.verificationMode: certificate -opensearch.ssl.certificateAuthorities: ["/usr/share/opensearch-dashboards/config/certs/root-ca.pem"] - -# Authentication - proxy auth from nginx (primary) + basic auth for kibanaserver -# Note: OpenSearch Dashboards doesn't support env var interpolation in YAML -# In production, use a secrets manager or pre-process this file -opensearch.username: "kibanaserver" -opensearch.password: "admin" -opensearch.requestHeadersAllowlist: ["securitytenant", "Authorization", "x-forwarded-for", "x-proxy-user", "x-proxy-roles"] - -# Proxy Authentication Configuration -# Nginx sets x-proxy-user/x-proxy-roles headers after validating user session via auth_request. -# Dashboards trusts these headers (no login form). Users must log in via the main app first. -# The kibanaserver user (above) is still used for Dashboards' own backend connection to OpenSearch. -opensearch_security.auth.type: "proxy" -opensearch_security.proxycache.user_header: "x-proxy-user" -opensearch_security.proxycache.roles_header: "x-proxy-roles" - -# Security Plugin Configuration - SaaS Multitenancy -# Each customer gets their own isolated tenant - no shared data by default -# Tenant is forced via nginx securitytenant header (per-org), no tenant picker shown -opensearch_security.multitenancy.enabled: true -opensearch_security.multitenancy.tenants.enable_global: false -opensearch_security.multitenancy.tenants.enable_private: false -opensearch_security.multitenancy.tenants.preferred: ["Custom"] -opensearch_security.multitenancy.show_roles: false -opensearch_security.multitenancy.enable_filter: false -opensearch_security.readonly_mode.roles: ["kibana_read_only"] -opensearch_security.cookie.secure: true -opensearch_security.cookie.isSameSite: "Strict" - -# Session configuration -opensearch_security.session.ttl: 3600000 -opensearch_security.session.keepalive: true - -# Default landing page - Discover instead of Home (which shows all plugin links) -uiSettings.overrides.defaultRoute: "/app/discover" - -# Logging -logging.dest: stdout -logging.silent: false -logging.quiet: false -logging.verbose: false - -# CSP headers for security -csp.strict: true -csp.warnLegacyBrowsers: true diff --git a/docker/opensearch-dashboards.yml b/docker/opensearch-dashboards.yml deleted file mode 100644 index 7c24007a3..000000000 --- a/docker/opensearch-dashboards.yml +++ /dev/null @@ -1,33 +0,0 @@ -# OpenSearch Dashboards configuration -# Mount this file to /usr/share/opensearch-dashboards/config/opensearch_dashboards.yml -# -# SECURITY NOTE: -# - Local development: Security plugin is disabled (DISABLE_SECURITY_DASHBOARDS_PLUGIN=true in docker-compose) -# - Production: Enable security plugin and configure multitenancy: -# 1. Remove DISABLE_SECURITY_PLUGIN=true from OpenSearch -# 2. Remove DISABLE_SECURITY_DASHBOARDS_PLUGIN=true from Dashboards -# 3. Configure TLS certificates and authentication -# 4. Add: opensearch_security.multitenancy.enabled: true -# 5. Add: opensearch_security.multitenancy.tenants.preferred: ["Private", "Global"] - -server.host: "0.0.0.0" -server.port: 5601 - -# Base path configuration for reverse proxy -server.basePath: "/analytics" -server.rewriteBasePath: true - -# OpenSearch connection -opensearch.hosts: ["http://opensearch:9200"] - -# Logging -logging.dest: stdout -logging.silent: false -logging.quiet: false -logging.verbose: false - -# Default landing page - Discover instead of Home (which shows all plugin links) -uiSettings.overrides.defaultRoute: "/app/discover" - -# CSP - relaxed for development (inline scripts needed by dashboards) -csp.strict: false diff --git a/docker/opensearch-init.sh b/docker/opensearch-init.sh deleted file mode 100755 index 1d9d97b16..000000000 --- a/docker/opensearch-init.sh +++ /dev/null @@ -1,102 +0,0 @@ -#!/bin/bash -# OpenSearch Dashboards initialization script -# Creates default index patterns and saved objects -# -# Environment variables: -# OPENSEARCH_DASHBOARDS_URL - Dashboards URL (default: http://opensearch-dashboards:5601) -# OPENSEARCH_SECURITY_ENABLED - Enable security mode (default: false) -# OPENSEARCH_ADMIN_PASSWORD - Admin password (not used with proxy auth, kept for reference) -# OPENSEARCH_CA_CERT - Path to CA cert for TLS (optional, for https) - -set -e - -# Note: Use /analytics prefix since dashboards is configured with server.basePath=/analytics -DASHBOARDS_URL="${OPENSEARCH_DASHBOARDS_URL:-http://opensearch-dashboards:5601}" -DASHBOARDS_BASE_PATH="/analytics" -MAX_RETRIES=30 -RETRY_INTERVAL=5 -SECURITY_ENABLED="${OPENSEARCH_SECURITY_ENABLED:-false}" - -# Wrapper function for authenticated curl requests -# When security is enabled, Dashboards uses proxy auth (not basic auth) -# We send x-proxy-user and x-proxy-roles headers to authenticate -auth_curl() { - if [ "$SECURITY_ENABLED" = "true" ]; then - curl -H "x-proxy-user: admin" -H "x-proxy-roles: admin,all_access" "$@" - else - curl "$@" - fi -} - -echo "[opensearch-init] Security mode: ${SECURITY_ENABLED}" -echo "[opensearch-init] Waiting for OpenSearch Dashboards to be ready..." - -# Wait for Dashboards to be healthy (use basePath) -# Accept 200 or 401 as "ready" - 401 means server is up but requires auth -# Note: Don't use -f flag as we want to capture 4xx status codes without curl failing -for i in $(seq 1 $MAX_RETRIES); do - HTTP_CODE=$(curl -s -o /dev/null -w '%{http_code}' "${DASHBOARDS_URL}${DASHBOARDS_BASE_PATH}/api/status" 2>/dev/null || echo "000") - - if [ "$HTTP_CODE" = "200" ] || [ "$HTTP_CODE" = "401" ]; then - echo "[opensearch-init] OpenSearch Dashboards is ready! (HTTP $HTTP_CODE)" - break - fi - - if [ $i -eq $MAX_RETRIES ]; then - echo "[opensearch-init] ERROR: OpenSearch Dashboards not ready after $((MAX_RETRIES * RETRY_INTERVAL)) seconds (last HTTP code: $HTTP_CODE)" - exit 1 - fi - - echo "[opensearch-init] Waiting for Dashboards... (attempt $i/$MAX_RETRIES, HTTP $HTTP_CODE)" - sleep $RETRY_INTERVAL -done - -# In secure mode, skip index pattern creation via Dashboards API -# Reason: Dashboards uses proxy auth which requires requests to come through nginx -# Index patterns will be created when users first access Dashboards through the normal flow -if [ "$SECURITY_ENABLED" = "true" ]; then - echo "[opensearch-init] Security mode enabled - skipping index pattern creation" - echo "[opensearch-init] Index patterns will be created on first user access via nginx" - echo "[opensearch-init] Initialization complete!" - exit 0 -fi - -# Check if index pattern already exists (insecure mode only) -echo "[opensearch-init] Checking for existing index patterns..." -EXISTING=$(auth_curl -sf "${DASHBOARDS_URL}${DASHBOARDS_BASE_PATH}/api/saved_objects/_find?type=index-pattern&search_fields=title&search=security-findings-*" \ - -H "osd-xsrf: true" 2>/dev/null || echo '{"total":0}') - -TOTAL=$(echo "$EXISTING" | grep -o '"total":[0-9]*' | grep -o '[0-9]*' || echo "0") - -if [ "$TOTAL" -gt 0 ]; then - echo "[opensearch-init] Index pattern 'security-findings-*' already exists, skipping creation" -else - echo "[opensearch-init] Creating index pattern 'security-findings-*'..." - - # Use specific ID so dashboards can reference it consistently - RESPONSE=$(auth_curl -sf -X POST "${DASHBOARDS_URL}${DASHBOARDS_BASE_PATH}/api/saved_objects/index-pattern/security-findings-*" \ - -H "Content-Type: application/json" \ - -H "osd-xsrf: true" \ - -d '{ - "attributes": { - "title": "security-findings-*", - "timeFieldName": "@timestamp" - } - }' 2>&1) - - if echo "$RESPONSE" | grep -q '"type":"index-pattern"'; then - echo "[opensearch-init] Successfully created index pattern 'security-findings-*'" - else - echo "[opensearch-init] WARNING: Failed to create index pattern. Response: $RESPONSE" - # Don't fail - the pattern might be created later when data exists - fi -fi - -# Set as default index pattern (optional, helps UX) -echo "[opensearch-init] Setting default index pattern..." -auth_curl -sf -X POST "${DASHBOARDS_URL}${DASHBOARDS_BASE_PATH}/api/opensearch-dashboards/settings" \ - -H "Content-Type: application/json" \ - -H "osd-xsrf: true" \ - -d '{"changes":{"defaultIndex":"security-findings-*"}}' > /dev/null 2>&1 || true - -echo "[opensearch-init] Initialization complete!" diff --git a/docker/opensearch-security/action_groups.yml b/docker/opensearch-security/action_groups.yml deleted file mode 100644 index 1b4a5179a..000000000 --- a/docker/opensearch-security/action_groups.yml +++ /dev/null @@ -1,61 +0,0 @@ -# OpenSearch Security - Action Groups -# -# Action groups bundle permissions together for easier role assignment. -# Most common groups are built-in, but custom ones can be defined here. -# -# Built-in action groups (no need to redefine): -# - cluster_all, cluster_monitor, cluster_composite_ops, cluster_composite_ops_ro -# - indices_all, indices_monitor, read, write, delete, create_index, manage -# - kibana_all_read, kibana_all_write -# -# Reference: https://opensearch.org/docs/latest/security/access-control/default-action-groups/ - ---- -_meta: - type: "actiongroups" - config_version: 2 - -# ============================================================================= -# CUSTOM ACTION GROUPS -# ============================================================================= - -# Index management for security findings -security_findings_write: - reserved: false - static: false - allowed_actions: - - "indices:data/write/index" - - "indices:data/write/bulk*" - - "indices:data/write/update" - - "indices:data/write/delete" - - "indices:admin/create" - - "indices:admin/mapping/put" - description: "Write access to security findings indices" - -security_findings_read: - reserved: false - static: false - allowed_actions: - - "indices:data/read/search*" - - "indices:data/read/get*" - - "indices:data/read/mget*" - - "indices:data/read/msearch*" - - "indices:data/read/scroll*" - - "indices:admin/mappings/get" - - "indices:admin/resolve/index" - description: "Read access to security findings indices" - -# Dashboard access for customers -dashboards_read: - reserved: false - static: false - allowed_actions: - - "kibana_all_read" - description: "Read-only access to dashboards" - -dashboards_write: - reserved: false - static: false - allowed_actions: - - "kibana_all_write" - description: "Write access to dashboards" diff --git a/docker/opensearch-security/allowlist.yml b/docker/opensearch-security/allowlist.yml deleted file mode 100644 index cd934c4ab..000000000 --- a/docker/opensearch-security/allowlist.yml +++ /dev/null @@ -1,13 +0,0 @@ -# OpenSearch Security - API Allowlist -# -# Controls which REST APIs can be accessed. -# Disabled by default (all APIs allowed based on role permissions). - ---- -_meta: - type: "allowlist" - config_version: 2 - -config: - enabled: false - requests: {} diff --git a/docker/opensearch-security/audit.yml b/docker/opensearch-security/audit.yml deleted file mode 100644 index f8fae3211..000000000 --- a/docker/opensearch-security/audit.yml +++ /dev/null @@ -1,30 +0,0 @@ -# OpenSearch Security - Audit Configuration -# -# Audit logging configuration for security events. - ---- -_meta: - type: "audit" - config_version: 2 - -config: - # Enable audit logging - enabled: true - audit: - # Log successful authentication - enable_rest: true - # Log transport layer (disabled for dev to reduce noise) - enable_transport: false - # What to log - resolve_bulk_requests: false - log_request_body: false - resolve_indices: true - exclude_sensitive_headers: true - # Ignore system indices - ignore_users: - - "kibanaserver" - ignore_requests: - - "SearchRequest" - - "indices:data/read/*" - compliance: - enabled: false diff --git a/docker/opensearch-security/config.yml b/docker/opensearch-security/config.yml deleted file mode 100644 index 7af3c434f..000000000 --- a/docker/opensearch-security/config.yml +++ /dev/null @@ -1,47 +0,0 @@ -# OpenSearch Security Configuration -# -# This file configures authentication domains for the security plugin. -# Proxy auth is used for nginx-authenticated requests (Dashboards access). -# Basic auth is used for direct API access (admin, worker). - ---- -_meta: - type: "config" - config_version: 2 - -config: - dynamic: - http: - xff: - enabled: true - # Trusted proxy IPs - templated at container start by docker-entrypoint-security.sh - # Default matches Docker bridge network (172.x.x.x) - internalProxies: '__INTERNAL_PROXIES__' - remoteIpHeader: 'X-Forwarded-For' - - authc: - # Proxy authentication for nginx-authenticated requests - # Nginx sets x-proxy-user and x-proxy-roles headers after auth validation - proxy_auth_domain: - http_enabled: true - transport_enabled: true - order: 0 - http_authenticator: - type: proxy - challenge: false - config: - user_header: "x-proxy-user" - roles_header: "x-proxy-roles" - authentication_backend: - type: noop - - # Basic auth fallback for direct API access (admin, worker) - basic_internal_auth_domain: - http_enabled: true - transport_enabled: true - order: 1 - http_authenticator: - type: basic - challenge: true - authentication_backend: - type: intern diff --git a/docker/opensearch-security/docker-entrypoint-security.sh b/docker/opensearch-security/docker-entrypoint-security.sh deleted file mode 100755 index 83314aab0..000000000 --- a/docker/opensearch-security/docker-entrypoint-security.sh +++ /dev/null @@ -1,108 +0,0 @@ -#!/bin/sh -# OpenSearch Security Entrypoint (Production-Ready) -# -# This entrypoint: -# 1. Templates the internalProxies regex in config.yml -# 2. Launches a background process to initialize security after OpenSearch starts -# 3. Uses a marker file to avoid re-initializing on every restart -# -# Environment variables: -# OPENSEARCH_INTERNAL_PROXIES - Trusted proxy IP regex (default: Docker bridge) -# SECURITY_AUTO_INIT - Auto-initialize security index (default: true) - -set -e - -# Configuration -INTERNAL_PROXIES="${OPENSEARCH_INTERNAL_PROXIES:-(172|192|10)\\.\\d+\\.\\d+\\.\\d+}" -SECURITY_AUTO_INIT="${SECURITY_AUTO_INIT:-true}" -SECURITY_INIT_MARKER="/usr/share/opensearch/data/.security_initialized" - -SRC_CONFIG="/usr/share/opensearch/config/opensearch-security/config.yml" -DEST_DIR="/usr/share/opensearch/config/opensearch-security-templated" -DEST_CONFIG="${DEST_DIR}/config.yml" - -echo "[opensearch-security] Templating internalProxies: ${INTERNAL_PROXIES}" - -if [ -f "${SRC_CONFIG}" ]; then - # Create destination directory if needed - mkdir -p "${DEST_DIR}" - - # Copy and template the config file - sed "s/__INTERNAL_PROXIES__/${INTERNAL_PROXIES}/g" "${SRC_CONFIG}" > "${DEST_CONFIG}" - - # Copy other security config files to the templated directory - for file in /usr/share/opensearch/config/opensearch-security/*.yml; do - filename=$(basename "$file") - if [ "$filename" != "config.yml" ]; then - cp "$file" "${DEST_DIR}/${filename}" - fi - done - - echo "[opensearch-security] Config templating complete" -else - echo "[opensearch-security] WARNING: Config file not found at ${SRC_CONFIG}" -fi - -# Background security initialization function -security_init_background() { - # Wait for OpenSearch to be ready - echo "[opensearch-security] Waiting for OpenSearch to be ready..." - ADMIN_PASSWORD="${OPENSEARCH_ADMIN_PASSWORD:-admin}" - MAX_RETRIES=60 - RETRY_COUNT=0 - - while [ $RETRY_COUNT -lt $MAX_RETRIES ]; do - # Use admin credentials - OpenSearch rejects unauthenticated requests - # even before security is fully initialized - if curl -sf -u "admin:${ADMIN_PASSWORD}" \ - --cacert /usr/share/opensearch/config/certs/root-ca.pem \ - https://localhost:9200/_cluster/health > /dev/null 2>&1; then - echo "[opensearch-security] OpenSearch is ready" - break - fi - RETRY_COUNT=$((RETRY_COUNT + 1)) - sleep 2 - done - - if [ $RETRY_COUNT -eq $MAX_RETRIES ]; then - echo "[opensearch-security] ERROR: OpenSearch not ready after $MAX_RETRIES attempts" - return 1 - fi - - # Always run securityadmin.sh to apply our templated config. - # OpenSearch may auto-init security from the raw config dir (with __INTERNAL_PROXIES__ - # placeholder), so we must overwrite it with the properly templated version. - # The marker file (checked at the outer level) prevents re-runs on subsequent restarts. - echo "[opensearch-security] Applying templated security config with securityadmin.sh..." - /usr/share/opensearch/plugins/opensearch-security/tools/securityadmin.sh \ - -cd "${DEST_DIR}" \ - -icl \ - -nhnv \ - -cacert /usr/share/opensearch/config/certs/root-ca.pem \ - -cert /usr/share/opensearch/config/certs/admin.pem \ - -key /usr/share/opensearch/config/certs/admin-key.pem - - if [ $? -eq 0 ]; then - echo "[opensearch-security] Security initialization complete" - touch "$SECURITY_INIT_MARKER" - else - echo "[opensearch-security] ERROR: Security initialization failed" - return 1 - fi -} - -# Launch background security initialization if enabled and not already done -if [ "${SECURITY_AUTO_INIT}" = "true" ]; then - if [ -f "$SECURITY_INIT_MARKER" ]; then - echo "[opensearch-security] Security previously initialized (marker exists)" - else - echo "[opensearch-security] Will initialize security after OpenSearch starts..." - # Run in background so OpenSearch can start - security_init_background & - fi -else - echo "[opensearch-security] Auto-init disabled (SECURITY_AUTO_INIT=${SECURITY_AUTO_INIT})" -fi - -# Execute the original OpenSearch entrypoint -exec /usr/share/opensearch/opensearch-docker-entrypoint.sh "$@" diff --git a/docker/opensearch-security/internal_users.yml b/docker/opensearch-security/internal_users.yml deleted file mode 100644 index fdb93d833..000000000 --- a/docker/opensearch-security/internal_users.yml +++ /dev/null @@ -1,72 +0,0 @@ -# OpenSearch Security - Internal Users (SaaS Model) -# -# USER PROVISIONING STRATEGY: -# Customer users are created dynamically via the Security REST API -# when users are added to the platform. This file only contains -# system users required for platform operations. -# -# Customer user creation example (via backend): -# PUT /_plugins/_security/api/internalusers/{user_email} -# { -# "password": "hashed_password", -# "backend_roles": ["customer_{customer_id}"], -# "attributes": { -# "customer_id": "{customer_id}", -# "email": "{user_email}" -# } -# } -# -# Password hashing: -# docker run -it opensearchproject/opensearch:2.11.1 \ -# /usr/share/opensearch/plugins/opensearch-security/tools/hash.sh -p - ---- -_meta: - type: "internalusers" - config_version: 2 - -# ============================================================================= -# SYSTEM USERS (Platform Operations) -# ============================================================================= - -# Platform admin - for internal operations only -admin: - # Default password: admin (CHANGE IN PRODUCTION!) - hash: "$2a$12$VcCDgh2NDk07JGN0rjGbM.Ad41qVR/YFJcgHp0UGns5JDymv..TOG" - reserved: true - backend_roles: - - "admin" - attributes: - role: "system" - description: "Platform administrator - internal use only" - -# Dashboards server user - used by OpenSearch Dashboards -kibanaserver: - # Default password: admin (matches OPENSEARCH_DASHBOARDS_PASSWORD default) - hash: "$2a$12$VcCDgh2NDk07JGN0rjGbM.Ad41qVR/YFJcgHp0UGns5JDymv..TOG" - reserved: true - attributes: - role: "system" - description: "Dashboards backend communication user" - -# Worker service user - for indexing security findings from worker processes -worker: - # Default password: worker (CHANGE IN PRODUCTION!) - hash: "$2a$12$VcCDgh2NDk07JGN0rjGbM.Ad41qVR/YFJcgHp0UGns5JDymv..TOG" - reserved: false - backend_roles: - - "worker_write" - attributes: - role: "system" - description: "Worker service for indexing security findings" - -# ============================================================================= -# CUSTOMER USERS -# Note: Customer users are created dynamically by the backend when users -# register or are invited to the platform. -# -# Each customer user will have: -# - backend_roles: ["customer_{customer_id}"] -# - attributes.customer_id: their customer ID -# - Mapped to customer-specific role for index isolation -# ============================================================================= diff --git a/docker/opensearch-security/nodes_dn.yml b/docker/opensearch-security/nodes_dn.yml deleted file mode 100644 index 566555f15..000000000 --- a/docker/opensearch-security/nodes_dn.yml +++ /dev/null @@ -1,12 +0,0 @@ -# OpenSearch Security - Node Distinguished Names -# -# For single-node development, this file is empty. -# In production multi-node clusters, list the DNs of all nodes. - ---- -_meta: - type: "nodesdn" - config_version: 2 - -# Allow all nodes with certificates signed by our CA -# (In production, specify exact node DNs for tighter security) diff --git a/docker/opensearch-security/roles.yml b/docker/opensearch-security/roles.yml deleted file mode 100644 index f2d44c4d7..000000000 --- a/docker/opensearch-security/roles.yml +++ /dev/null @@ -1,177 +0,0 @@ -# OpenSearch Security - Roles Configuration (SaaS Model) -# -# INDEX ISOLATION STRATEGY: -# Each customer's data is stored in indices prefixed with their customer ID: -# {customer_id}-analytics-* -# {customer_id}-workflows-* -# {customer_id}-scans-* -# -# Roles are created dynamically per customer with index patterns that -# restrict access to only their data. This file defines role templates -# and system roles. -# -# Dynamic role creation example (via backend): -# PUT /_plugins/_security/api/roles/customer_{customer_id} -# { -# "cluster_permissions": ["cluster_composite_ops_ro"], -# "index_permissions": [{ -# "index_patterns": ["{customer_id}-*"], -# "allowed_actions": ["read", "indices:data/read/*"] -# }], -# "tenant_permissions": [{ -# "tenant_patterns": ["{customer_id}"], -# "allowed_actions": ["kibana_all_write"] -# }] -# } - ---- -_meta: - type: "roles" - config_version: 2 - -# ============================================================================= -# SYSTEM ROLES (Platform Operations) -# ============================================================================= - -# Platform admin - full access for operators -platform_admin: - reserved: true - cluster_permissions: - - "*" - index_permissions: - - index_patterns: - - "*" - allowed_actions: - - "*" - tenant_permissions: - - tenant_patterns: - - "*" - allowed_actions: - - "kibana_all_write" - -# Worker write role - for indexing security findings from worker processes -# Write-only access to security-findings-* indices (no read of other orgs' data) -worker_write: - reserved: false - description: "Worker service role for indexing security findings" - cluster_permissions: - - "cluster_composite_ops_ro" - - "indices:data/write/*" - index_permissions: - - index_patterns: - - "security-findings-*" - allowed_actions: - - "write" - - "create_index" - - "indices:data/write/*" - - "indices:admin/mapping/put" - -# ============================================================================= -# CUSTOMER ROLE TEMPLATE -# These are templates - actual roles are created dynamically per customer -# ============================================================================= - -# Template: Customer read-write access (for active users) -# Actual role name: customer_{customer_id}_rw -# Index pattern: {customer_id}-* -customer_template_rw: - reserved: false - description: "Template for customer read-write roles - DO NOT USE DIRECTLY" - cluster_permissions: - - "cluster_composite_ops_ro" - - "indices:data/read/scroll*" - # Required for Dashboards saved objects (bulk writes to .kibana_* tenant indices) - - "indices:data/write/bulk" - # Alerting: monitor CRUD, execution, alerts, and destinations (legacy endpoints) - - "cluster:admin/opendistro/alerting/monitor/get" - - "cluster:admin/opendistro/alerting/monitor/search" - - "cluster:admin/opendistro/alerting/monitor/write" - - "cluster:admin/opendistro/alerting/monitor/execute" - - "cluster:admin/opendistro/alerting/alerts/get" - - "cluster:admin/opendistro/alerting/alerts/ack" - - "cluster:admin/opendistro/alerting/destination/get" - - "cluster:admin/opendistro/alerting/destination/write" - - "cluster:admin/opendistro/alerting/destination/delete" - # Notifications plugin (OpenSearch 2.x): channel features + config CRUD - - "cluster:admin/opensearch/notifications/features" - - "cluster:admin/opensearch/notifications/configs/get" - - "cluster:admin/opensearch/notifications/configs/create" - - "cluster:admin/opensearch/notifications/configs/update" - - "cluster:admin/opensearch/notifications/configs/delete" - index_permissions: - - index_patterns: - - "CUSTOMER_ID_PLACEHOLDER-*" - allowed_actions: - - "read" - - "write" - - "create_index" - - "indices:data/read/*" - - "indices:data/write/*" - - "indices:admin/mapping/put" - - index_patterns: - - ".kibana*" - allowed_actions: - - "read" - - "write" - - "create_index" - - "indices:data/read/*" - - "indices:data/write/*" - - "indices:admin/mapping/put" - tenant_permissions: - - tenant_patterns: - - "CUSTOMER_ID_PLACEHOLDER" - allowed_actions: - - "kibana_all_write" - -# Template: Customer read-only access (for viewers) -# Actual role name: customer_{customer_id}_ro -# Index pattern: {customer_id}-* -customer_template_ro: - reserved: false - description: "Template for customer read-only roles - DO NOT USE DIRECTLY" - cluster_permissions: - - "cluster_composite_ops_ro" - # Required for Dashboards saved objects (bulk writes to .kibana_* tenant indices) - - "indices:data/write/bulk" - # Alerting: monitor CRUD, execution, alerts, and destinations (legacy endpoints) - - "cluster:admin/opendistro/alerting/monitor/get" - - "cluster:admin/opendistro/alerting/monitor/search" - - "cluster:admin/opendistro/alerting/monitor/write" - - "cluster:admin/opendistro/alerting/monitor/execute" - - "cluster:admin/opendistro/alerting/alerts/get" - - "cluster:admin/opendistro/alerting/alerts/ack" - - "cluster:admin/opendistro/alerting/destination/get" - - "cluster:admin/opendistro/alerting/destination/write" - - "cluster:admin/opendistro/alerting/destination/delete" - # Notifications plugin (OpenSearch 2.x): channel features + config CRUD - - "cluster:admin/opensearch/notifications/features" - - "cluster:admin/opensearch/notifications/configs/get" - - "cluster:admin/opensearch/notifications/configs/create" - - "cluster:admin/opensearch/notifications/configs/update" - - "cluster:admin/opensearch/notifications/configs/delete" - index_permissions: - - index_patterns: - - "CUSTOMER_ID_PLACEHOLDER-*" - allowed_actions: - - "read" - - "indices:data/read/*" - - index_patterns: - - ".kibana*" - allowed_actions: - - "read" - - "write" - - "create_index" - - "indices:data/read/*" - - "indices:data/write/*" - - "indices:admin/mapping/put" - tenant_permissions: - - tenant_patterns: - - "CUSTOMER_ID_PLACEHOLDER" - allowed_actions: - - "kibana_all_write" - -# ============================================================================= -# DASHBOARDS INTERNAL ROLES -# Note: kibana_server and kibana_read_only are built-in static roles -# Do NOT redefine them here - they are managed by OpenSearch Security plugin -# ============================================================================= diff --git a/docker/opensearch-security/roles_mapping.yml b/docker/opensearch-security/roles_mapping.yml deleted file mode 100644 index 69636259d..000000000 --- a/docker/opensearch-security/roles_mapping.yml +++ /dev/null @@ -1,73 +0,0 @@ -# OpenSearch Security - Roles Mapping (SaaS Model) -# -# DYNAMIC ROLE MAPPING: -# Customer role mappings are created dynamically when users are provisioned. -# Each customer user is mapped to their customer-specific role. -# -# Example dynamic mapping creation (via backend): -# PUT /_plugins/_security/api/rolesmapping/customer_{customer_id}_rw -# { -# "users": ["user@customer.com"], -# "backend_roles": ["customer_{customer_id}"] -# } -# -# The backend should: -# 1. Create customer tenant when customer onboards -# 2. Create customer role (customer_{id}_rw or customer_{id}_ro) -# 3. Map user to customer role when user is added - ---- -_meta: - type: "rolesmapping" - config_version: 2 - -# ============================================================================= -# SYSTEM ROLE MAPPINGS -# ============================================================================= - -# Platform admin mapping - internal operators only -platform_admin: - reserved: true - users: - - "admin" - backend_roles: - - "platform_admin" - description: "Platform administrators with full system access" - -# Dashboards server mapping -kibana_server: - reserved: true - users: - - "kibanaserver" - description: "OpenSearch Dashboards server user" - -# Security REST API access - for admin operations -security_rest_api_access: - reserved: true - users: - - "admin" - backend_roles: - - "platform_admin" - description: "Access to Security REST API for tenant/role management" - -# Worker service mapping - for indexing security findings -worker_write: - reserved: false - users: - - "worker" - backend_roles: - - "worker_write" - description: "Worker service for indexing security findings" - -# ============================================================================= -# CUSTOMER ROLE MAPPINGS -# Note: Customer-specific mappings are created dynamically by the backend -# when customers and users are provisioned. -# -# Pattern for dynamic mappings: -# Role: customer_{customer_id}_rw -# Users: [list of customer's users with write access] -# -# Role: customer_{customer_id}_ro -# Users: [list of customer's users with read-only access] -# ============================================================================= diff --git a/docker/opensearch-security/tenants.yml b/docker/opensearch-security/tenants.yml deleted file mode 100644 index ae9d03937..000000000 --- a/docker/opensearch-security/tenants.yml +++ /dev/null @@ -1,28 +0,0 @@ -# OpenSearch Security - Tenants Configuration (SaaS Model) -# -# TENANT ISOLATION STRATEGY: -# Each customer gets their own isolated tenant and index pattern. -# No shared/global dashboards - sharing is explicitly opt-in. -# -# Tenants are created dynamically via the Security REST API when -# a new customer is onboarded. Tenant name = customer ID. -# -# Index naming convention: {customer_id}-analytics-* -# Each customer's role restricts access to only their indices. -# -# Example dynamic tenant creation (via backend): -# POST /_plugins/_security/api/tenants/{customer_id} -# { "description": "Tenant for customer {customer_id}" } - ---- -_meta: - type: "tenants" - config_version: 2 - -# NOTE: Customer tenants are created dynamically by the application backend -# when customers are onboarded. This file only contains system tenants. - -# Admin tenant - for platform operators only (not customers) -__platform_admin: - reserved: true - description: "Platform administration - internal use only" diff --git a/docker/opensearch-security/whitelist.yml b/docker/opensearch-security/whitelist.yml deleted file mode 100644 index cb55f2a96..000000000 --- a/docker/opensearch-security/whitelist.yml +++ /dev/null @@ -1,13 +0,0 @@ -# OpenSearch Security - API Whitelist (legacy name for allowlist) -# -# This file is required by securityadmin.sh even in OpenSearch 2.x. -# Actual configuration is in allowlist.yml. - ---- -_meta: - type: 'whitelist' - config_version: 2 - -config: - enabled: false - requests: {} diff --git a/docker/opensearch.dev-secure.yml b/docker/opensearch.dev-secure.yml deleted file mode 100644 index f9f0494ac..000000000 --- a/docker/opensearch.dev-secure.yml +++ /dev/null @@ -1,35 +0,0 @@ -# OpenSearch Development Configuration with Security -# Mount to: /usr/share/opensearch/config/opensearch.yml - -cluster.name: shipsec-dev-secure -node.name: opensearch-node1 -network.host: 0.0.0.0 - -# Single-node mode -discovery.type: single-node -bootstrap.memory_lock: true - -# Security Plugin Configuration -plugins.security.ssl.transport.pemcert_filepath: certs/node.pem -plugins.security.ssl.transport.pemkey_filepath: certs/node-key.pem -plugins.security.ssl.transport.pemtrustedcas_filepath: certs/root-ca.pem -plugins.security.ssl.transport.enforce_hostname_verification: false - -plugins.security.ssl.http.enabled: true -plugins.security.ssl.http.pemcert_filepath: certs/node.pem -plugins.security.ssl.http.pemkey_filepath: certs/node-key.pem -plugins.security.ssl.http.pemtrustedcas_filepath: certs/root-ca.pem - -plugins.security.allow_unsafe_democertificates: false -plugins.security.allow_default_init_securityindex: true - -# Admin DN - Required for securityadmin.sh and REST API access -plugins.security.authcz.admin_dn: - - "CN=admin,OU=ShipSec,O=ShipSecAI,L=SF,ST=CA,C=US" - -plugins.security.audit.type: internal_opensearch -plugins.security.enable_snapshot_restore_privilege: true -plugins.security.check_snapshot_restore_write_privileges: true -plugins.security.restapi.roles_enabled: - - "all_access" - - "security_rest_api_access" diff --git a/docker/redpanda-console-config.yaml b/docker/redpanda-console-config.yaml deleted file mode 100644 index fbb0702a4..000000000 --- a/docker/redpanda-console-config.yaml +++ /dev/null @@ -1,11 +0,0 @@ -kafka: - brokers: - - redpanda:9092 - schemaRegistry: - enabled: false - -redpanda: - adminApi: - enabled: true - urls: - - http://redpanda:9644 diff --git a/docker/scripts/generate-certs.sh b/docker/scripts/generate-certs.sh deleted file mode 100755 index 4a04c2fd7..000000000 --- a/docker/scripts/generate-certs.sh +++ /dev/null @@ -1,91 +0,0 @@ -#!/bin/bash -# Generate TLS certificates for OpenSearch production deployment -# -# This script creates: -# - Root CA certificate and key -# - Node certificate for OpenSearch (server) -# - Admin certificate for cluster management -# -# Usage: ./generate-certs.sh [output-dir] -# -# Requirements: openssl - -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -OUTPUT_DIR="${1:-$SCRIPT_DIR/../certs}" -DAYS_VALID=365 - -# Certificate Subject fields -COUNTRY="US" -STATE="CA" -LOCALITY="SF" -ORGANIZATION="ShipSecAI" -ORG_UNIT="ShipSec" - -echo "=== OpenSearch Certificate Generator ===" -echo "Output directory: $OUTPUT_DIR" -echo "" - -# Create output directory -mkdir -p "$OUTPUT_DIR" -cd "$OUTPUT_DIR" - -# Check if certificates already exist -if [[ -f "root-ca.pem" ]]; then - echo "WARNING: Certificates already exist in $OUTPUT_DIR" - read -p "Overwrite existing certificates? (y/N): " -n 1 -r - echo - if [[ ! $REPLY =~ ^[Yy]$ ]]; then - echo "Aborted." - exit 1 - fi -fi - -echo "1. Generating Root CA..." -openssl genrsa -out root-ca-key.pem 2048 -openssl req -new -x509 -sha256 -key root-ca-key.pem -out root-ca.pem -days $DAYS_VALID \ - -subj "/C=$COUNTRY/ST=$STATE/L=$LOCALITY/O=$ORGANIZATION/OU=$ORG_UNIT/CN=Root CA" - -echo "2. Generating Admin Certificate..." -openssl genrsa -out admin-key-temp.pem 2048 -openssl pkcs8 -inform PEM -outform PEM -in admin-key-temp.pem -topk8 -nocrypt -out admin-key.pem -openssl req -new -key admin-key.pem -out admin.csr \ - -subj "/C=$COUNTRY/ST=$STATE/L=$LOCALITY/O=$ORGANIZATION/OU=$ORG_UNIT/CN=admin" -openssl x509 -req -in admin.csr -CA root-ca.pem -CAkey root-ca-key.pem -CAcreateserial \ - -sha256 -out admin.pem -days $DAYS_VALID -rm admin-key-temp.pem admin.csr - -echo "3. Generating Node Certificate..." -# Create extension file for SAN (Subject Alternative Names) -cat > node-ext.cnf << EOF -subjectAltName = DNS:localhost, DNS:opensearch, DNS:opensearch-node1, IP:127.0.0.1 -EOF - -openssl genrsa -out node-key-temp.pem 2048 -openssl pkcs8 -inform PEM -outform PEM -in node-key-temp.pem -topk8 -nocrypt -out node-key.pem -openssl req -new -key node-key.pem -out node.csr \ - -subj "/C=$COUNTRY/ST=$STATE/L=$LOCALITY/O=$ORGANIZATION/OU=$ORG_UNIT/CN=opensearch-node1" -openssl x509 -req -in node.csr -CA root-ca.pem -CAkey root-ca-key.pem -CAcreateserial \ - -sha256 -out node.pem -days $DAYS_VALID -extfile node-ext.cnf -rm node-key-temp.pem node.csr node-ext.cnf - -echo "4. Setting permissions..." -chmod 600 *-key.pem -chmod 644 *.pem - -echo "" -echo "=== Certificates Generated Successfully ===" -echo "" -echo "Files created in $OUTPUT_DIR:" -ls -la "$OUTPUT_DIR" -echo "" -echo "Next steps:" -echo " 1. Review the certificates" -echo " 2. Set OPENSEARCH_ADMIN_PASSWORD and OPENSEARCH_DASHBOARDS_PASSWORD environment variables" -echo " 3. Run: docker compose -f docker-compose.infra.yml -f docker-compose.prod.yml up -d" -echo "" -echo "For production deployments:" -echo " - Use proper certificate authority (e.g., Let's Encrypt, internal CA)" -echo " - Store private keys securely (e.g., HashiCorp Vault, AWS Secrets Manager)" -echo " - Rotate certificates before expiration ($DAYS_VALID days)" diff --git a/docker/scripts/hash-password.sh b/docker/scripts/hash-password.sh deleted file mode 100755 index 22ba22ce2..000000000 --- a/docker/scripts/hash-password.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash -# Generate BCrypt password hash for OpenSearch Security internal users -# -# Usage: ./hash-password.sh [password] -# -# If password is not provided, it will be read from stdin (useful for piping) -# The hash can be used in opensearch-security/internal_users.yml -# -# Example: -# ./hash-password.sh mySecurePassword123 -# echo "myPassword" | ./hash-password.sh - -set -euo pipefail - -OPENSEARCH_IMAGE="${OPENSEARCH_IMAGE:-opensearchproject/opensearch:2.11.1}" - -if [ $# -ge 1 ]; then - PASSWORD="$1" -elif [ ! -t 0 ]; then - # Read from stdin if piped - read -r PASSWORD -else - # Interactive prompt - echo -n "Enter password to hash: " >&2 - read -rs PASSWORD - echo >&2 -fi - -if [ -z "$PASSWORD" ]; then - echo "Error: Password cannot be empty" >&2 - exit 1 -fi - -# Use OpenSearch's built-in hash.sh tool to generate BCrypt hash -docker run --rm -i "$OPENSEARCH_IMAGE" \ - /usr/share/opensearch/plugins/opensearch-security/tools/hash.sh \ - -p "$PASSWORD" 2>/dev/null | tail -1 diff --git a/docker/scripts/security-init.sh b/docker/scripts/security-init.sh deleted file mode 100755 index 0e8cd3e03..000000000 --- a/docker/scripts/security-init.sh +++ /dev/null @@ -1,157 +0,0 @@ -#!/bin/bash -# Initialize OpenSearch Security index using securityadmin.sh -# -# This script properly initializes the security configuration without using -# the deprecated demo installer. It should be run: -# - After first-time OpenSearch startup -# - After modifying security configuration files -# - When migrating from demo to production security -# -# Prerequisites: -# - OpenSearch must be running with TLS enabled -# - Admin certificates must exist in docker/certs/ -# - Security config files in docker/opensearch-security/ -# -# Usage: -# ./security-init.sh # Use defaults -# ./security-init.sh --force # Force reinitialize (overwrites existing) -# OPENSEARCH_HOST=my-host ./security-init.sh # Custom host - -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -DOCKER_DIR="$SCRIPT_DIR/.." - -# Configuration -OPENSEARCH_HOST="${OPENSEARCH_HOST:-opensearch}" -OPENSEARCH_PORT="${OPENSEARCH_PORT:-9200}" -CERTS_DIR="${CERTS_DIR:-$DOCKER_DIR/certs}" -SECURITY_CONFIG_DIR="${SECURITY_CONFIG_DIR:-$DOCKER_DIR/opensearch-security}" -CONTAINER_NAME="${OPENSEARCH_CONTAINER:-shipsec-opensearch}" - -# Parse arguments -FORCE_INIT=false -while [[ $# -gt 0 ]]; do - case $1 in - --force|-f) - FORCE_INIT=true - shift - ;; - *) - echo "Unknown option: $1" - exit 1 - ;; - esac -done - -echo "=== OpenSearch Security Initialization ===" -echo "" -echo "Configuration:" -echo " Container: $CONTAINER_NAME" -echo " Certs dir: $CERTS_DIR" -echo " Security dir: $SECURITY_CONFIG_DIR" -echo " Force init: $FORCE_INIT" -echo "" - -# Verify prerequisites -if [ ! -f "$CERTS_DIR/admin.pem" ] || [ ! -f "$CERTS_DIR/admin-key.pem" ]; then - echo "Error: Admin certificates not found in $CERTS_DIR" - echo "Run: just generate-certs" - exit 1 -fi - -if [ ! -f "$CERTS_DIR/root-ca.pem" ]; then - echo "Error: Root CA certificate not found in $CERTS_DIR" - exit 1 -fi - -if [ ! -d "$SECURITY_CONFIG_DIR" ]; then - echo "Error: Security config directory not found: $SECURITY_CONFIG_DIR" - exit 1 -fi - -# Check if OpenSearch container is running -if ! docker ps --filter "name=$CONTAINER_NAME" --format "{{.Names}}" | grep -q "$CONTAINER_NAME"; then - echo "Error: OpenSearch container '$CONTAINER_NAME' is not running" - echo "Start it first with: just dev or just prod-secure" - exit 1 -fi - -# Wait for OpenSearch to be ready -echo "Waiting for OpenSearch to be ready..." -MAX_RETRIES=30 -for i in $(seq 1 $MAX_RETRIES); do - if docker exec "$CONTAINER_NAME" curl -sf \ - --cacert /usr/share/opensearch/config/certs/root-ca.pem \ - https://localhost:9200/_cluster/health > /dev/null 2>&1; then - echo "OpenSearch is ready!" - break - fi - - if [ $i -eq $MAX_RETRIES ]; then - echo "Error: OpenSearch not ready after $MAX_RETRIES attempts" - exit 1 - fi - - echo " Waiting... (attempt $i/$MAX_RETRIES)" - sleep 2 -done - -# Check if security index already exists -echo "" -echo "Checking security index status..." -SECURITY_STATUS=$(docker exec "$CONTAINER_NAME" curl -sf \ - --cacert /usr/share/opensearch/config/certs/root-ca.pem \ - https://localhost:9200/_plugins/_security/health 2>/dev/null || echo "not_initialized") - -if echo "$SECURITY_STATUS" | grep -q '"status":"UP"'; then - if [ "$FORCE_INIT" != "true" ]; then - echo "Security index already initialized." - echo "Use --force to reinitialize (this will overwrite existing configuration)" - exit 0 - fi - echo "Security index exists, but --force specified. Reinitializing..." -else - echo "Security index not initialized. Proceeding with initialization..." -fi - -# Copy security config files to container (in case they've been updated) -echo "" -echo "Copying security configuration to container..." -docker cp "$SECURITY_CONFIG_DIR/." "$CONTAINER_NAME:/usr/share/opensearch/config/opensearch-security-init/" - -# Run securityadmin.sh -echo "" -echo "Running securityadmin.sh to initialize security index..." -docker exec "$CONTAINER_NAME" /usr/share/opensearch/plugins/opensearch-security/tools/securityadmin.sh \ - -cd /usr/share/opensearch/config/opensearch-security-init \ - -icl \ - -nhnv \ - -cacert /usr/share/opensearch/config/certs/root-ca.pem \ - -cert /usr/share/opensearch/config/certs/admin.pem \ - -key /usr/share/opensearch/config/certs/admin-key.pem - -# Verify initialization -echo "" -echo "Verifying security initialization..." -sleep 2 -FINAL_STATUS=$(docker exec "$CONTAINER_NAME" curl -sf \ - --cacert /usr/share/opensearch/config/certs/root-ca.pem \ - https://localhost:9200/_plugins/_security/health 2>/dev/null || echo "{}") - -if echo "$FINAL_STATUS" | grep -q '"status":"UP"'; then - echo "" - echo "=== Security Initialization Complete ===" - echo "" - echo "Security plugin status: UP" - echo "" - echo "Next steps:" - echo " - Test authentication: curl -u admin:PASSWORD --cacert docker/certs/root-ca.pem https://localhost:9200" - echo " - Update internal_users.yml with production password hashes" - echo " - Re-run this script with --force after updating passwords" -else - echo "" - echo "Warning: Security initialization may have failed" - echo "Check OpenSearch logs: docker logs $CONTAINER_NAME" - exit 1 -fi diff --git a/install.sh b/install.sh deleted file mode 100644 index af05357f6..000000000 --- a/install.sh +++ /dev/null @@ -1,1528 +0,0 @@ -#!/usr/bin/env bash -# install.sh - One-liner installer for ShipSec Studio (Production/Docker mode) -# -# Usage: -# curl -fsSL https://raw.githubusercontent.com/ShipSecAI/studio/main/install.sh | bash -# -# This script installs ShipSec Studio using pre-built Docker images from GHCR. -# For development setup, see: https://github.com/ShipSecAI/studio#option-3-development-setup -# -# Supported platforms: macOS, Linux, Windows (Git Bash/MSYS2/WSL) - -set -u -o pipefail -# Note: We intentionally keep the default IFS (space, tab, newline) -# because we use space-separated lists for MISSING_DEPS and INSTALL_FAILED - -# ---------- Config ---------- -REPO_URL="https://github.com/ShipSecAI/studio" -REPO_DIR="studio" -WAIT_DOCKER_SEC=60 - -# ---------- Colors ---------- -setup_colors() { - if [[ -t 1 ]] && [[ -n "${TERM:-}" ]]; then - GREEN='\033[0;32m' - YELLOW='\033[1;33m' - RED='\033[0;31m' - CYAN='\033[0;36m' - BLUE='\033[0;34m' - BOLD='\033[1m' - NC='\033[0m' - else - GREEN='' - YELLOW='' - RED='' - CYAN='' - BLUE='' - BOLD='' - NC='' - fi -} -setup_colors - -# ---------- Logging ---------- -# Note: Using %b to interpret escape sequences in the argument -log() { printf "\n${GREEN}==>${NC} ${BOLD}%s${NC}\n" "$1"; } -info() { printf " %b\n" "$1"; } -warn() { printf " ${YELLOW}Warning:${NC} %b\n" "$1"; } -err() { printf " ${RED}Error:${NC} %b\n" "$1"; } - -# ---------- Traps ---------- -on_err() { - local rc=$? - printf "\n" - err "Installation failed (exit code: $rc)" - err "If you need help, please visit: https://github.com/ShipSecAI/studio/issues" - exit $rc -} -on_int() { - printf "\n" - warn "Installation cancelled by user." - exit 130 -} -trap 'on_err' ERR -trap 'on_int' INT - -# ---------- Utility ---------- -command_exists() { command -v "$1" >/dev/null 2>&1; } - -# Check if we can interact with user (even when piped via curl | bash) -is_interactive() { - # stdin is a terminal - [ -t 0 ] && return 0 - # stdin is piped but /dev/tty exists (curl | bash scenario) - [ -e /dev/tty ] && return 0 - # Truly non-interactive - return 1 -} - -# Cross-platform user input -# Uses /dev/tty to allow prompts even when script is piped (curl | bash) -ask_yes_no() { - local prompt="$1" - local default="${2:-n}" - local yn_hint - - if [ "$default" = "y" ]; then - yn_hint="[Y/n]" - else - yn_hint="[y/N]" - fi - - # Check if we can read from /dev/tty (works even when script is piped) - if [ -t 0 ]; then - # stdin is a terminal - : - elif [ -e /dev/tty ]; then - # stdin is piped but /dev/tty exists - we can still prompt - exec < /dev/tty - else - # Truly non-interactive (no terminal available) - case "$default" in - y|Y) return 0 ;; - *) return 1 ;; - esac - fi - - while true; do - printf " %s %s " "$prompt" "$yn_hint" - read -r ans || ans="" - ans="${ans:-$default}" - case "$ans" in - y|Y|yes|YES|Yes) return 0 ;; - n|N|no|NO|No) return 1 ;; - *) printf " Please enter 'y' for yes or 'n' for no.\n" ;; - esac - done -} - -# ---------- Platform Detection ---------- -detect_platform() { - local os_raw - os_raw="$(uname -s 2>/dev/null || echo Unknown)" - - case "$os_raw" in - Darwin) - PLATFORM="macos" - PLATFORM_NAME="macOS" - ;; - Linux) - if grep -qEi "(microsoft|wsl)" /proc/version 2>/dev/null; then - PLATFORM="wsl" - PLATFORM_NAME="Windows (WSL)" - else - PLATFORM="linux" - PLATFORM_NAME="Linux" - fi - ;; - MINGW*|MSYS*|CYGWIN*) - PLATFORM="windows" - PLATFORM_NAME="Windows (Git Bash)" - ;; - *) - PLATFORM="unknown" - PLATFORM_NAME="Unknown" - ;; - esac -} - -# ---------- Dependency Installation ---------- - -# Check if we can use sudo -can_sudo() { - if command_exists sudo; then - # Check if user can sudo without password or is root - if [ "$(id -u)" = "0" ] || sudo -n true 2>/dev/null; then - return 0 - fi - # In interactive mode, try to get sudo access (let user see password prompt) - if is_interactive; then - info "Some operations require administrator privileges." - if sudo -v; then - return 0 - fi - fi - fi - return 1 -} - -# Check if current user is in docker group (for Linux/WSL) -check_docker_group() { - # Skip on macOS and Windows - they don't use docker group - if [ "$PLATFORM" = "macos" ] || [ "$PLATFORM" = "windows" ]; then - return 0 - fi - - # Root doesn't need docker group - if [ "$(id -u)" = "0" ]; then - return 0 - fi - - # Check if docker group exists and user is a member - if command_exists docker && getent group docker >/dev/null 2>&1; then - if ! groups 2>/dev/null | grep -qw docker; then - return 1 # User not in docker group - fi - fi - - return 0 -} - -# Install Docker - always asks for permission -install_docker() { - log "Installing Docker" - - printf "\n" - warn "Docker installation requires your permission." - printf "\n" - - case "$PLATFORM" in - macos) - info "On macOS, you can install Docker in two ways:" - printf "\n" - info " ${BOLD} Option 1: Docker Desktop${NC} (GUI app, easiest)" - info " ${BOLD} Option 2: Colima${NC} (CLI-only, lightweight)" - printf "\n" - - if ! ask_yes_no "Would you like to install Docker now?" "y"; then - info "Docker installation skipped." - show_install_instructions "docker" - return 1 - fi - - if command_exists brew; then - printf "\n" - info "Which Docker runtime would you prefer?" - printf "\n" - info " 1) Docker Desktop (GUI application)" - info " 2) Colima (CLI-only, runs in terminal)" - printf "\n" - - local choice="" - if is_interactive; then - printf " Enter choice [1/2]: " - read -r choice || choice="1" - else - choice="1" - fi - - case "$choice" in - 2) - info "Installing Colima and Docker CLI via Homebrew..." - printf "\n" - if brew install colima docker docker-compose; then - printf "\n" - info "${GREEN}Colima and Docker CLI installed successfully!${NC}" - info "Starting Colima..." - if colima start; then - info "${GREEN}Colima is running! Docker daemon is ready.${NC}" - return 0 - else - warn "Colima installed but failed to start. Try: colima start" - return 0 - fi - else - err "Failed to install Colima" - return 1 - fi - ;; - *) - info "Installing Docker Desktop via Homebrew..." - printf "\n" - if brew install --cask docker; then - printf "\n" - info "${GREEN}Docker Desktop installed successfully!${NC}" - return 0 - else - err "Failed to install Docker via Homebrew" - return 1 - fi - ;; - esac - else - printf "\n" - warn "Homebrew is not installed." - info "Please install Docker Desktop manually from:" - printf "\n" - printf " https://www.docker.com/products/docker-desktop\n" - printf "\n" - info "Or install Homebrew first:" - printf "\n" - printf " /bin/bash -c \"\$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)\"\n" - printf "\n" - return 1 - fi - ;; - linux) - if ! ask_yes_no "Would you like to install Docker Engine now?" "y"; then - info "Docker installation skipped." - show_install_instructions "docker" - return 1 - fi - - if can_sudo; then - info "Installing Docker Engine via official script..." - printf "\n" - if curl -fsSL https://get.docker.com | sudo sh; then - printf "\n" - info "Adding current user to docker group..." - sudo usermod -aG docker "$USER" 2>/dev/null || true - printf "\n" - printf "${GREEN}┌─────────────────────────────────────────────────────────────────┐${NC}\n" - printf "${GREEN}│${NC} ${BOLD}🚨 Docker installed but requires logout/login${NC} ${GREEN}│${NC}\n" - printf "${GREEN}│${NC} ${GREEN}│${NC}\n" - printf "${GREEN}│${NC} Your user has been added to the 'docker' group, but this ${GREEN}│${NC}\n" - printf "${GREEN}│${NC} change won't take effect until you log out and back in. ${GREEN}│${NC}\n" - printf "${GREEN}│${NC} ${GREEN}│${NC}\n" - printf "${GREEN}│${NC} ${BOLD}➡️ Please log out, log back in, then run:${NC} ${GREEN}│${NC}\n" - printf "${GREEN}│${NC} ${GREEN}│${NC}\n" - printf "${GREEN}│${NC} curl -fsSL https://raw.githubusercontent.com/ShipSecAI/ ${GREEN}│${NC}\n" - printf "${GREEN}│${NC} studio/main/install.sh | bash ${GREEN}│${NC}\n" - printf "${GREEN}│${NC} ${GREEN}│${NC}\n" - printf "${GREEN}└─────────────────────────────────────────────────────────────────┘${NC}\n" - printf "\n" - info "Exiting now to avoid permission issues." - exit 0 - else - err "Failed to install Docker" - return 1 - fi - else - printf "\n" - err "Cannot install Docker without sudo access." - info "Please install Docker manually:" - printf "\n" - printf " curl -fsSL https://get.docker.com | sudo sh\n" - printf " sudo usermod -aG docker \$USER\n" - printf "\n" - return 1 - fi - ;; - wsl) - printf "\n" - info "For WSL, you have two options:" - printf "\n" - info " ${BOLD} Option 1: Docker Desktop for Windows (Recommended)${NC}" - info " - Install Docker Desktop from: https://www.docker.com/products/docker-desktop" - info " - Enable WSL2 integration in Docker Desktop Settings > Resources > WSL Integration" - printf "\n" - info " ${BOLD} Option 2: Docker Engine in WSL${NC}" - - if ! ask_yes_no "Would you like to install Docker Engine directly in WSL?" "y"; then - info "Docker installation skipped." - info "Please install Docker Desktop for Windows and enable WSL2 integration." - return 1 - fi - - if can_sudo; then - info "Installing Docker Engine..." - printf "\n" - if curl -fsSL https://get.docker.com | sudo sh; then - printf "\n" - info "Adding current user to docker group..." - sudo usermod -aG docker "$USER" 2>/dev/null || true - printf "\n" - printf "${GREEN}┌─────────────────────────────────────────────────────────────────┐${NC}\n" - printf "${GREEN}│${NC} ${BOLD}🚨 Docker installed but requires logout/login${NC} ${GREEN}│${NC}\n" - printf "${GREEN}│${NC} ${GREEN}│${NC}\n" - printf "${GREEN}│${NC} Your user has been added to the 'docker' group, but this ${GREEN}│${NC}\n" - printf "${GREEN}│${NC} change won't take effect until you log out and back in. ${GREEN}│${NC}\n" - printf "${GREEN}│${NC} ${GREEN}│${NC}\n" - printf "${GREEN}│${NC} ${BOLD}➡️ Please log out, log back in, then run:${NC} ${GREEN}│${NC}\n" - printf "${GREEN}│${NC} ${GREEN}│${NC}\n" - printf "${GREEN}│${NC} curl -fsSL https://raw.githubusercontent.com/ShipSecAI/ ${GREEN}│${NC}\n" - printf "${GREEN}│${NC} studio/main/install.sh | bash ${GREEN}│${NC}\n" - printf "${GREEN}│${NC} ${GREEN}│${NC}\n" - printf "${GREEN}└─────────────────────────────────────────────────────────────────┘${NC}\n" - printf "\n" - info "Exiting now to avoid permission issues." - exit 0 - else - err "Failed to install Docker" - return 1 - fi - else - err "Cannot install Docker without sudo access." - return 1 - fi - ;; - windows) - printf "\n" - info "Docker Desktop for Windows is required." - printf "\n" - - if ! ask_yes_no "Would you like to install Docker Desktop now?" "y"; then - info "Docker installation skipped." - show_install_instructions "docker" - return 1 - fi - - if command_exists winget; then - info "Installing Docker Desktop via winget..." - printf "\n" - if winget install Docker.DockerDesktop --accept-source-agreements --accept-package-agreements; then - printf "\n" - info "${GREEN}Docker Desktop installed successfully!${NC}" - info "Please restart your terminal and run this script again." - return 0 - else - err "Failed to install Docker Desktop" - return 1 - fi - fi - - info "Please install Docker Desktop manually from:" - printf "\n" - printf " https://www.docker.com/products/docker-desktop\n" - printf "\n" - return 1 - ;; - *) - err "Automatic Docker installation not supported on this platform." - info "Please install Docker manually." - return 1 - ;; - esac -} - -# Install just automatically -install_just() { - log "Installing just" - - case "$PLATFORM" in - macos) - if command_exists brew; then - info "Installing just via Homebrew..." - if brew install just; then - info "${GREEN}just installed successfully!${NC}" - return 0 - else - err "Failed to install just" - return 1 - fi - else - warn "Homebrew is not installed. Installing just via script..." - mkdir -p ~/.local/bin - # Use --force to overwrite if already exists - if curl --proto '=https' --tlsv1.2 -sSf https://just.systems/install.sh | bash -s -- --to ~/.local/bin --force; then - export PATH="$HOME/.local/bin:$PATH" - info "${GREEN}just installed to ~/.local/bin${NC}" - warn "Add ~/.local/bin to your PATH permanently by adding this to your shell profile:" - printf " export PATH=\"\$HOME/.local/bin:\$PATH\"\n" - return 0 - else - err "Failed to install just" - return 1 - fi - fi - ;; - linux|wsl) - info "Installing just via official script..." - mkdir -p ~/.local/bin - # Use --force to overwrite if already exists - if curl --proto '=https' --tlsv1.2 -sSf https://just.systems/install.sh | bash -s -- --to ~/.local/bin --force; then - # Add to PATH for current session - export PATH="$HOME/.local/bin:$PATH" - info "${GREEN}just installed to ~/.local/bin${NC}" - - # Check if ~/.local/bin is already in shell profile - local shell_profile="" - if [ -n "${BASH_VERSION:-}" ]; then - shell_profile="$HOME/.bashrc" - elif [ -n "${ZSH_VERSION:-}" ]; then - shell_profile="$HOME/.zshrc" - elif [ -f "$HOME/.bashrc" ]; then - shell_profile="$HOME/.bashrc" - elif [ -f "$HOME/.profile" ]; then - shell_profile="$HOME/.profile" - fi - - # Add to shell profile if not already there - if [ -n "$shell_profile" ] && [ -f "$shell_profile" ]; then - if ! grep -q '\.local/bin' "$shell_profile" 2>/dev/null; then - printf '\n# Added by ShipSec Studio installer\nexport PATH="$HOME/.local/bin:$PATH"\n' >> "$shell_profile" - info "Added ~/.local/bin to PATH in $shell_profile" - fi - else - warn "Add ~/.local/bin to your PATH permanently by adding this to your shell profile:" - printf " export PATH=\"\$HOME/.local/bin:\$PATH\"\n" - fi - return 0 - else - err "Failed to install just" - return 1 - fi - ;; - windows) - if command_exists scoop; then - info "Installing just via Scoop..." - if scoop install just; then - info "${GREEN}just installed successfully!${NC}" - return 0 - fi - elif command_exists choco; then - info "Installing just via Chocolatey..." - if choco install just -y; then - info "${GREEN}just installed successfully!${NC}" - return 0 - fi - fi - - err "Could not install just automatically." - info "Please install just manually from: https://github.com/casey/just/releases" - return 1 - ;; - *) - err "Automatic just installation not supported on this platform." - return 1 - ;; - esac -} - -# Install jq automatically -install_jq() { - log "Installing jq" - - case "$PLATFORM" in - macos) - if command_exists brew; then - info "Installing jq via Homebrew..." - if brew install jq; then - info "${GREEN}jq installed successfully!${NC}" - return 0 - fi - fi - err "Failed to install jq" - return 1 - ;; - linux|wsl) - if can_sudo; then - # Detect package manager - if command_exists apt-get; then - info "Installing jq via apt..." - if sudo apt-get update -qq && sudo apt-get install -y jq; then - info "${GREEN}jq installed successfully!${NC}" - return 0 - fi - elif command_exists dnf; then - info "Installing jq via dnf..." - if sudo dnf install -y jq; then - info "${GREEN}jq installed successfully!${NC}" - return 0 - fi - elif command_exists yum; then - info "Installing jq via yum..." - if sudo yum install -y jq; then - info "${GREEN}jq installed successfully!${NC}" - return 0 - fi - elif command_exists pacman; then - info "Installing jq via pacman..." - if sudo pacman -S --noconfirm jq; then - info "${GREEN}jq installed successfully!${NC}" - return 0 - fi - fi - fi - err "Failed to install jq" - return 1 - ;; - windows) - if command_exists scoop; then - info "Installing jq via Scoop..." - if scoop install jq; then - info "${GREEN}jq installed successfully!${NC}" - return 0 - fi - elif command_exists choco; then - info "Installing jq via Chocolatey..." - if choco install jq -y; then - info "${GREEN}jq installed successfully!${NC}" - return 0 - fi - fi - err "Failed to install jq" - return 1 - ;; - *) - err "Automatic jq installation not supported on this platform." - return 1 - ;; - esac -} - -# Install git automatically -install_git() { - log "Installing git" - - case "$PLATFORM" in - macos) - info "Installing git via Xcode Command Line Tools..." - if xcode-select --install 2>/dev/null; then - info "Please complete the Xcode Command Line Tools installation and run this script again." - return 1 - elif command_exists brew; then - info "Installing git via Homebrew..." - if brew install git; then - info "${GREEN}git installed successfully!${NC}" - return 0 - fi - fi - err "Failed to install git" - return 1 - ;; - linux|wsl) - if can_sudo; then - if command_exists apt-get; then - info "Installing git via apt..." - if sudo apt-get update -qq && sudo apt-get install -y git; then - info "${GREEN}git installed successfully!${NC}" - return 0 - fi - elif command_exists dnf; then - info "Installing git via dnf..." - if sudo dnf install -y git; then - info "${GREEN}git installed successfully!${NC}" - return 0 - fi - elif command_exists yum; then - info "Installing git via yum..." - if sudo yum install -y git; then - info "${GREEN}git installed successfully!${NC}" - return 0 - fi - elif command_exists pacman; then - info "Installing git via pacman..." - if sudo pacman -S --noconfirm git; then - info "${GREEN}git installed successfully!${NC}" - return 0 - fi - fi - fi - err "Failed to install git" - return 1 - ;; - windows) - if command_exists winget; then - info "Installing git via winget..." - if winget install Git.Git --accept-source-agreements --accept-package-agreements; then - info "${GREEN}git installed successfully!${NC}" - info "Please restart your terminal for git to be available." - return 0 - fi - fi - err "Failed to install git" - info "Please install git from: https://git-scm.com/download/win" - return 1 - ;; - *) - err "Automatic git installation not supported on this platform." - return 1 - ;; - esac -} - -# Install curl automatically -install_curl() { - log "Installing curl" - - case "$PLATFORM" in - macos) - # curl is pre-installed on macOS - if command_exists brew; then - info "Installing curl via Homebrew..." - if brew install curl; then - info "${GREEN}curl installed successfully!${NC}" - return 0 - fi - fi - err "Failed to install curl" - return 1 - ;; - linux|wsl) - if can_sudo; then - if command_exists apt-get; then - info "Installing curl via apt..." - if sudo apt-get update -qq && sudo apt-get install -y curl; then - info "${GREEN}curl installed successfully!${NC}" - return 0 - fi - elif command_exists dnf; then - info "Installing curl via dnf..." - if sudo dnf install -y curl; then - info "${GREEN}curl installed successfully!${NC}" - return 0 - fi - elif command_exists yum; then - info "Installing curl via yum..." - if sudo yum install -y curl; then - info "${GREEN}curl installed successfully!${NC}" - return 0 - fi - fi - fi - err "Failed to install curl" - return 1 - ;; - windows) - # curl is included in Windows 10+ and Git Bash - if command_exists choco; then - info "Installing curl via Chocolatey..." - if choco install curl -y; then - info "${GREEN}curl installed successfully!${NC}" - return 0 - fi - fi - err "Failed to install curl" - return 1 - ;; - *) - err "Automatic curl installation not supported on this platform." - return 1 - ;; - esac -} - -# Try to install a missing dependency -try_install_dep() { - local dep="$1" - - case "$dep" in - docker) install_docker ;; - just) install_just ;; - jq) install_jq ;; - git) install_git ;; - curl) install_curl ;; - *) return 1 ;; - esac -} - -# Start Docker daemon -start_docker_daemon() { - log "Starting Docker daemon" - - case "$PLATFORM" in - macos) - # Check for Colima first (CLI-based, can start from terminal) - if command_exists colima; then - info "Found Colima - starting Docker runtime from terminal..." - printf "\n" - - # Check if Colima is already running - if colima status 2>/dev/null | grep -q "Running"; then - info "${GREEN}Colima is already running!${NC}" - return 0 - fi - - info "Starting Colima..." - if colima start 2>&1; then - printf "\n" - # Wait for Docker to be ready - printf " Waiting for Docker to be ready" - local start=$(date +%s) - while ! docker info >/dev/null 2>&1; do - local now=$(date +%s) - local elapsed=$((now - start)) - if [ "$elapsed" -ge "$WAIT_DOCKER_SEC" ]; then - printf "\n\n" - err "Docker did not become ready within ${WAIT_DOCKER_SEC} seconds." - return 1 - fi - printf "." - sleep 2 - done - printf " ${GREEN}ready!${NC}\n" - return 0 - else - warn "Failed to start Colima, trying Docker Desktop..." - fi - fi - - # Try Docker Desktop - check multiple possible locations - local docker_app="" - if [ -d "/Applications/Docker.app" ]; then - docker_app="/Applications/Docker.app" - elif [ -d "$HOME/Applications/Docker.app" ]; then - docker_app="$HOME/Applications/Docker.app" - fi - - if [ -n "$docker_app" ]; then - info "Starting Docker Desktop..." - open -g "$docker_app" - - printf " Waiting for Docker to be ready" - local start=$(date +%s) - while ! docker info >/dev/null 2>&1; do - local now=$(date +%s) - local elapsed=$((now - start)) - if [ "$elapsed" -ge "$WAIT_DOCKER_SEC" ]; then - printf "\n\n" - err "Docker did not start within ${WAIT_DOCKER_SEC} seconds." - info "Docker Desktop may need to complete first-time setup." - info "Please open Docker Desktop manually from Applications, complete the setup," - info "then run this script again." - return 1 - fi - printf "." - sleep 2 - done - printf " ${GREEN}ready!${NC}\n" - return 0 - fi - - # Docker CLI exists but no runtime - user probably installed just 'docker' via brew - if command_exists docker; then - printf "\n" - warn "Docker CLI is installed, but no Docker runtime is running." - info "The 'docker' command needs a runtime (Docker Desktop or Colima) to work." - printf "\n" - - if is_interactive && command_exists brew; then - info "Would you like to install a Docker runtime now?" - printf "\n" - info " 1) Colima (CLI-only, lightweight, recommended)" - info " 2) Docker Desktop (GUI application)" - info " 3) Skip (I'll install manually)" - printf "\n" - - printf " Enter choice [1/2/3]: " - local choice="" - read -r choice || choice="3" - - case "$choice" in - 1) - info "Installing Colima..." - printf "\n" - if brew install colima docker-compose; then - info "${GREEN}Colima installed!${NC}" - info "Starting Colima..." - if colima start; then - info "${GREEN}Colima is running! Docker daemon is ready.${NC}" - return 0 - else - err "Failed to start Colima. Try running: colima start" - return 1 - fi - else - err "Failed to install Colima" - return 1 - fi - ;; - 2) - info "Installing Docker Desktop..." - printf "\n" - if brew install --cask docker; then - info "${GREEN}Docker Desktop installed!${NC}" - info "Starting Docker Desktop..." - open -g "/Applications/Docker.app" - - printf " Waiting for Docker to be ready" - local start=$(date +%s) - while ! docker info >/dev/null 2>&1; do - local now=$(date +%s) - local elapsed=$((now - start)) - if [ "$elapsed" -ge "$WAIT_DOCKER_SEC" ]; then - printf "\n\n" - warn "Docker Desktop is taking a while to start." - info "Please wait for Docker Desktop to finish starting, then run this script again." - return 1 - fi - printf "." - sleep 2 - done - printf " ${GREEN}ready!${NC}\n" - return 0 - else - err "Failed to install Docker Desktop" - return 1 - fi - ;; - *) - info "Skipping Docker runtime installation." - ;; - esac - fi - - printf "\n" - info "Please install a Docker runtime manually:" - printf "\n" - info " ${BOLD}Option 1: Colima (CLI-only, lightweight)${NC}" - printf " brew install colima docker-compose\n" - printf " colima start\n" - printf "\n" - info " ${BOLD}Option 2: Docker Desktop${NC}" - printf " brew install --cask docker\n" - printf " # Then open Docker Desktop from Applications\n" - printf "\n" - return 1 - fi - - # Neither Colima nor Docker Desktop found, and no docker CLI - err "No Docker installation found." - info "Please install Docker using one of these methods:" - printf "\n" - info " ${BOLD} Option 1: Colima (CLI-only, recommended for terminal users)${NC}" - printf " brew install colima docker docker-compose\n" - printf " colima start\n" - printf "\n" - info " ${BOLD} Option 2: Docker Desktop${NC}" - printf " brew install --cask docker\n" - printf " # Then open Docker Desktop from Applications\n" - printf "\n" - return 1 - ;; - linux) - if can_sudo; then - # Try systemctl first (systemd) - preferred method - if command_exists systemctl; then - info "Starting Docker via systemctl..." - if sudo systemctl start docker 2>/dev/null; then - # Wait for Docker to be ready (use sudo for docker info if needed) - printf " Waiting for Docker to be ready" - local start=$(date +%s) - while ! sudo docker info >/dev/null 2>&1; do - local now=$(date +%s) - local elapsed=$((now - start)) - if [ "$elapsed" -ge "$WAIT_DOCKER_SEC" ]; then - printf "\n\n" - err "Docker did not start within ${WAIT_DOCKER_SEC} seconds." - return 1 - fi - printf "." - sleep 2 - done - printf " ${GREEN}ready!${NC}\n" - - # Enable Docker to start on boot - sudo systemctl enable docker 2>/dev/null || true - return 0 - fi - fi - - # Try service command (SysVinit) - fallback for non-systemd systems - if command_exists service; then - info "Starting Docker via service command..." - if sudo service docker start 2>/dev/null; then - sleep 3 - if sudo docker info >/dev/null 2>&1; then - info "${GREEN}Docker daemon started!${NC}" - return 0 - fi - fi - fi - fi - - # No backgrounding dockerd - it's dangerous and conflicts with init systems - err "Failed to start Docker daemon." - info "Please start Docker manually using your system's init system:" - printf "\n" - printf " # For systemd (most modern Linux):\n" - printf " sudo systemctl start docker\n" - printf "\n" - printf " # For SysVinit:\n" - printf " sudo service docker start\n" - return 1 - ;; - wsl) - # In WSL, try service command - if can_sudo; then - if command_exists service; then - info "Starting Docker via service command..." - if sudo service docker start 2>/dev/null; then - sleep 3 - if sudo docker info >/dev/null 2>&1; then - info "${GREEN}Docker daemon started!${NC}" - return 0 - fi - fi - fi - fi - - # No backgrounding dockerd - it's dangerous - printf "\n" - warn "Could not start Docker daemon automatically in WSL." - info "Please use one of these options:" - printf "\n" - info " 1. Start Docker Desktop for Windows (with WSL2 integration enabled)" - info " 2. Start the Docker service manually: sudo service docker start" - return 1 - ;; - windows) - # On Windows (Git Bash), try to start Docker Desktop via cmd.exe - # This is more robust than hardcoding paths (handles non-C drives, localized Windows, etc.) - info "Attempting to start Docker Desktop..." - - if cmd.exe /c start "" "Docker Desktop" 2>/dev/null; then - printf " Waiting for Docker to be ready" - local start=$(date +%s) - while ! docker info >/dev/null 2>&1; do - local now=$(date +%s) - local elapsed=$((now - start)) - if [ "$elapsed" -ge "$WAIT_DOCKER_SEC" ]; then - printf "\n\n" - warn "Docker did not become ready within ${WAIT_DOCKER_SEC} seconds." - info "Docker Desktop may still be starting. Please wait and try again." - return 1 - fi - printf "." - sleep 2 - done - printf " ${GREEN}ready!${NC}\n" - return 0 - else - warn "Could not start Docker Desktop automatically." - info "Please start Docker Desktop manually from the Start menu." - return 1 - fi - ;; - *) - err "Cannot start Docker daemon on this platform automatically." - return 1 - ;; - esac -} - -# ---------- Dependency Installation Instructions (fallback) ---------- -show_install_instructions() { - local dep="$1" - - printf "\n" - printf " ${BOLD}How to install ${dep}:${NC}\n" - printf "\n" - - case "$dep" in - docker) - case "$PLATFORM" in - macos) - printf " ${CYAN} Option 1: Download Docker Desktop${NC}\n" - printf " https://www.docker.com/products/docker-desktop\n" - printf "\n" - printf " ${CYAN} Option 2: Install via Homebrew${NC}\n" - printf " brew install --cask docker\n" - ;; - linux) - printf " ${CYAN} Install Docker Engine:${NC}\n" - printf " curl -fsSL https://get.docker.com | sudo sh\n" - printf " sudo usermod -aG docker \$USER\n" - printf " # Log out and back in for group changes to take effect\n" - ;; - wsl) - printf " ${CYAN} Option 1: Use Docker Desktop for Windows${NC}\n" - printf " Install Docker Desktop and enable WSL2 integration in Settings\n" - printf " https://www.docker.com/products/docker-desktop\n" - printf "\n" - printf " ${CYAN} Option 2: Install Docker Engine in WSL${NC}\n" - printf " curl -fsSL https://get.docker.com | sudo sh\n" - printf " sudo usermod -aG docker \$USER\n" - ;; - windows) - printf " ${CYAN}Install Docker Desktop for Windows:${NC}\n" - printf " https://www.docker.com/products/docker-desktop\n" - printf "\n" - printf " ${CYAN}Or via winget:${NC}\n" - printf " winget install Docker.DockerDesktop\n" - ;; - esac - ;; - just) - case "$PLATFORM" in - macos) - printf " ${CYAN} Install via Homebrew:${NC}\n" - printf " brew install just\n" - ;; - linux|wsl) - printf " ${CYAN} Option 1: Install via script${NC}\n" - printf " curl --proto '=https' --tlsv1.2 -sSf https://just.systems/install.sh | bash -s -- --to ~/.local/bin\n" - printf " # Add ~/.local/bin to your PATH if not already\n" - printf "\n" - printf " ${CYAN} Option 2: Install via package manager${NC}\n" - printf " # Debian/Ubuntu (if available)\n" - printf " sudo apt install just\n" - ;; - windows) - printf " ${CYAN} Option 1: Install via Scoop${NC}\n" - printf " scoop install just\n" - printf "\n" - printf " ${CYAN} Option 2: Install via Chocolatey${NC}\n" - printf " choco install just\n" - printf "\n" - printf " ${CYAN} Option 3: Download from GitHub${NC}\n" - printf " https://github.com/casey/just/releases\n" - ;; - esac - ;; - curl) - case "$PLATFORM" in - macos) - printf " curl is pre-installed on macOS.\n" - printf " If missing, install via: brew install curl\n" - ;; - linux|wsl) - printf " ${CYAN}Debian/Ubuntu:${NC}\n" - printf " sudo apt-get update && sudo apt-get install -y curl\n" - printf "\n" - printf " ${CYAN}RHEL/CentOS/Fedora:${NC}\n" - printf " sudo dnf install curl\n" - ;; - windows) - printf " curl is included in Windows 10+ and Git Bash.\n" - printf " If missing, install via: choco install curl\n" - ;; - esac - ;; - jq) - case "$PLATFORM" in - macos) - printf " ${CYAN} Install via Homebrew:${NC}\n" - printf " brew install jq\n" - ;; - linux|wsl) - printf " ${CYAN} Debian/Ubuntu:${NC}\n" - printf " sudo apt-get update && sudo apt-get install -y jq\n" - printf "\n" - printf " ${CYAN} RHEL/CentOS/Fedora:${NC}\n" - printf " sudo dnf install jq\n" - ;; - windows) - printf " ${CYAN} Option 1: Install via Scoop${NC}\n" - printf " scoop install jq\n" - printf "\n" - printf " ${CYAN} Option 2: Install via Chocolatey${NC}\n" - printf " choco install jq\n" - ;; - esac - ;; - git) - case "$PLATFORM" in - macos) - printf " ${CYAN} Install via Xcode Command Line Tools:${NC}\n" - printf " xcode-select --install\n" - printf "\n" - printf " ${CYAN} Or via Homebrew:${NC}\n" - printf " brew install git\n" - ;; - linux|wsl) - printf " ${CYAN} Debian/Ubuntu:${NC}\n" - printf " sudo apt-get update && sudo apt-get install -y git\n" - printf "\n" - printf " ${CYAN} RHEL/CentOS/Fedora:${NC}\n" - printf " sudo dnf install git\n" - ;; - windows) - printf " ${CYAN} Download Git for Windows:${NC}\n" - printf " https://git-scm.com/download/win\n" - printf "\n" - printf " ${CYAN} Or via winget:${NC}\n" - printf " winget install Git.Git\n" - ;; - esac - ;; - esac -} - -# ---------- Main Script ---------- - -detect_platform - -# Banner -printf "\n" -printf "${BLUE}┌─────────────────────────────────────────────────────────────────┐${NC}\n" -printf "${BLUE}│${NC} ${BLUE}│${NC}\n" -printf "${BLUE}│${NC} ${BOLD}ShipSec Studio Installer${NC} ${BLUE}│${NC}\n" -printf "${BLUE}│${NC} Self-Hosted Production Deployment ${BLUE}│${NC}\n" -printf "${BLUE}│${NC} ${BLUE}│${NC}\n" -printf "${BLUE}└─────────────────────────────────────────────────────────────────┘${NC}\n" -printf "\n" -info "Platform: ${BOLD}$PLATFORM_NAME${NC}" -info "Documentation: https://docs.shipsec.ai" -printf "\n" - -# ---------- Early Check: Truly non-interactive mode without sudo ---------- -# Only warn if we truly can't interact (no /dev/tty available) -if ! is_interactive; then - # Truly non-interactive mode - check if we have sudo access - if [ "$PLATFORM" = "linux" ] || [ "$PLATFORM" = "wsl" ]; then - if [ "$(id -u)" != "0" ] && ! sudo -n true 2>/dev/null; then - warn "Running in non-interactive mode without sudo access." - info "If dependencies need to be installed, the script will fail." - info "For unattended installation, either:" - info " - Run as root" - info " - Configure passwordless sudo" - info " - Pre-install all dependencies (docker, just, curl, jq, git)" - printf "\n" - fi - fi -fi - -# ---------- Check Prerequisites ---------- -log "Checking prerequisites" -printf "\n" -info "ShipSec Studio requires the following tools:" -info " - docker (container runtime)" -info " - just (command runner)" -info " - curl (HTTP client)" -info " - jq (JSON processor)" -info " - git (version control)" -printf "\n" - -MISSING_DEPS="" -ALL_OK=true - -# Check each dependency (docker last since it may require logout/login on Linux) -for dep in just curl jq git docker; do - if command_exists "$dep"; then - case "$dep" in - docker) ver=$(docker --version 2>/dev/null | sed 's/Docker version //' | cut -d',' -f1) ;; - just) ver=$(just --version 2>/dev/null | head -1) ;; - curl) ver=$(curl --version 2>/dev/null | head -1 | awk '{print $2}') ;; - jq) ver=$(jq --version 2>/dev/null) ;; - git) ver=$(git --version 2>/dev/null | sed 's/git version //') ;; - esac - printf " ${GREEN}✓${NC} %-10s %s\n" "$dep" "$ver" - else - printf " ${RED}✗${NC} %-10s ${RED}not found${NC}\n" "$dep" - MISSING_DEPS="$MISSING_DEPS $dep" - ALL_OK=false - fi -done - -# If dependencies are missing, offer to install them -if [ "$ALL_OK" = false ]; then - MISSING_DEPS="${MISSING_DEPS# }" # trim leading space - - printf "\n" - warn "Missing required dependencies: ${BOLD}$MISSING_DEPS${NC}" - printf "\n" - - # Check if we can interact with user (works even with curl | bash) - if is_interactive; then - if ask_yes_no "Would you like to install the missing dependencies automatically?" "y"; then - INSTALL_FAILED="" - - for dep in $MISSING_DEPS; do - printf "\n" - if try_install_dep "$dep"; then - # Re-check if the command is now available - if command_exists "$dep"; then - printf " ${GREEN}✓${NC} $dep is now available\n" - else - # Some installations require PATH update or terminal restart - warn "$dep was installed but may require a terminal restart to be available." - INSTALL_FAILED="$INSTALL_FAILED $dep" - fi - else - INSTALL_FAILED="$INSTALL_FAILED $dep" - fi - done - - # Check if any installations failed - if [ -n "$INSTALL_FAILED" ]; then - INSTALL_FAILED="${INSTALL_FAILED# }" # trim leading space - printf "\n" - err "Could not install: ${BOLD}$INSTALL_FAILED${NC}" - printf "\n" - info "Please install these dependencies manually:" - - for dep in $INSTALL_FAILED; do - show_install_instructions "$dep" - done - - printf "\n" - info "After installing, run this script again:" - printf "\n" - printf " curl -fsSL https://raw.githubusercontent.com/ShipSecAI/studio/main/install.sh | bash\n" - printf "\n" - exit 1 - fi - - printf "\n" - info "${GREEN}All dependencies installed successfully!${NC}" - else - # User declined automatic installation - printf "\n" - info "Manual installation instructions:" - - for dep in $MISSING_DEPS; do - show_install_instructions "$dep" - done - - printf "\n" - info "After installing the missing dependencies, run this script again:" - printf "\n" - printf " curl -fsSL https://raw.githubusercontent.com/ShipSecAI/studio/main/install.sh | bash\n" - printf "\n" - exit 1 - fi - else - # Non-interactive mode - show instructions and exit - err "Missing required dependencies: ${BOLD}$MISSING_DEPS${NC}" - - for dep in $MISSING_DEPS; do - show_install_instructions "$dep" - done - - printf "\n" - info "After installing the missing dependencies, run this script again:" - printf "\n" - printf " curl -fsSL https://raw.githubusercontent.com/ShipSecAI/studio/main/install.sh | bash\n" - printf "\n" - exit 1 - fi -else - printf "\n" - info "${GREEN}All prerequisites are installed!${NC}" -fi - -# ---------- Check Docker Group Membership (Linux/WSL only) ---------- -if [ "$PLATFORM" = "linux" ] || [ "$PLATFORM" = "wsl" ]; then - if ! check_docker_group; then - printf "\n" - warn "You are not in the 'docker' group." - info "Docker group membership is required to run Docker commands without sudo." - printf "\n" - info "To fix this, run:" - printf "\n" - printf " sudo usermod -aG docker \$USER\n" - printf "\n" - info "Then ${BOLD}log out and log back in${NC} for the change to take effect." - info "After logging back in, run this script again." - printf "\n" - exit 1 - fi -fi - -# ---------- Check Docker Daemon ---------- -log "Checking Docker daemon" - -if ! docker info >/dev/null 2>&1; then - printf "\n" - warn "Docker daemon is not running." - printf "\n" - - # Check if we can interact with user (works even with curl | bash) - if is_interactive; then - if ask_yes_no "Would you like to start Docker automatically?" "y"; then - printf "\n" - if start_docker_daemon; then - printf "\n" - info "${GREEN}Docker daemon is now running!${NC}" - else - printf "\n" - err "Failed to start Docker daemon automatically." - printf "\n" - - case "$PLATFORM" in - macos) - info "Please start Docker Desktop from your Applications folder and run this script again." - ;; - linux) - info "Please start Docker manually:" - printf " sudo systemctl start docker\n" - printf "\n" - info "Then run this script again." - ;; - wsl) - info "To use Docker in WSL, you have two options:" - printf "\n" - info " 1. Start Docker Desktop for Windows (with WSL2 integration enabled)" - info " 2. Start the Docker service in WSL: sudo service docker start" - printf "\n" - info "Then run this script again." - ;; - windows) - info "Please start Docker Desktop for Windows and run this script again." - ;; - esac - exit 1 - fi - else - printf "\n" - case "$PLATFORM" in - macos) - info "Please start Docker Desktop from your Applications folder and run this script again." - ;; - linux) - info "To start Docker, run:" - printf "\n" - printf " sudo systemctl start docker\n" - printf "\n" - info "Then run this script again." - ;; - wsl) - info "To use Docker in WSL, you have two options:" - printf "\n" - info " 1. Start Docker Desktop for Windows (with WSL2 integration enabled)" - info " 2. Start the Docker service in WSL: sudo service docker start" - printf "\n" - info "Then run this script again." - ;; - windows) - info "Please start Docker Desktop for Windows and run this script again." - ;; - esac - exit 1 - fi - else - # Non-interactive mode - try to start automatically - printf "\n" - info "Attempting to start Docker daemon automatically..." - printf "\n" - - if start_docker_daemon; then - printf "\n" - info "${GREEN}Docker daemon is now running!${NC}" - else - printf "\n" - err "Failed to start Docker daemon." - printf "\n" - - case "$PLATFORM" in - macos) - info "Please start Docker Desktop and run this script again." - ;; - linux) - info "Please start Docker manually: sudo systemctl start docker" - ;; - wsl) - info "Please start Docker Desktop for Windows or run: sudo service docker start" - ;; - windows) - info "Please start Docker Desktop for Windows." - ;; - esac - exit 1 - fi - fi -else - printf "\n" - info "${GREEN}Docker daemon is running!${NC}" -fi - -# ---------- Repository Setup ---------- -log "Setting up repository" - -IN_REPO=false - -# Check if already in the repo -if [ -d .git ] && [ -f justfile ]; then - IN_REPO=true - info "Already in ShipSec Studio repository." -# Check if repo exists in current directory -elif [ -d "$REPO_DIR" ] && [ -d "$REPO_DIR/.git" ] && [ -f "$REPO_DIR/justfile" ]; then - info "Found existing repository in ./$REPO_DIR" - cd "$REPO_DIR" || { err "Failed to enter directory"; exit 1; } - IN_REPO=true -fi - -if [ "$IN_REPO" = false ]; then - if [ -d "$REPO_DIR" ]; then - printf "\n" - warn "Directory '$REPO_DIR' already exists." - - if ask_yes_no "Do you want to use the existing directory?" "y"; then - cd "$REPO_DIR" || { err "Failed to enter directory"; exit 1; } - else - info "Please remove or rename the '$REPO_DIR' directory and run this script again." - exit 1 - fi - else - printf "\n" - info "Cloning repository from GitHub..." - printf "\n" - - if ! git clone "$REPO_URL" "$REPO_DIR"; then - err "Failed to clone repository" - exit 1 - fi - - cd "$REPO_DIR" || { err "Failed to enter directory"; exit 1; } - fi -fi - -PROJECT_ROOT="$(pwd)" -printf "\n" -info "Project directory: ${BOLD}$PROJECT_ROOT${NC}" - -# ---------- Confirm Installation ---------- -log "Ready to install" - -printf "\n" -info "This will:" -info " 1. Fetch the latest release version from GitHub" -info " 2. Pull pre-built Docker images from GHCR" -info " 3. Start the full stack (frontend, backend, worker, infrastructure)" -printf "\n" -info "The following services will be available:" -info " - Frontend: http://localhost:8090" -info " - Backend: http://localhost:3211" -info " - Temporal UI: http://localhost:8081" -printf "\n" - -if ! ask_yes_no "Do you want to proceed with the installation?" "y"; then - printf "\n" - info "Installation cancelled." - printf "\n" - info "To install later, run:" - printf "\n" - printf " cd %s && just prod start-latest\n" "$PROJECT_ROOT" - printf "\n" - exit 0 -fi - -# ---------- Start Installation ---------- -log "Installing ShipSec Studio" - -printf "\n" -if ! just prod start-latest; then - printf "\n" - err "Installation failed." - err "Please check the error messages above." - printf "\n" - info "For troubleshooting, visit: https://github.com/ShipSecAI/studio/issues" - exit 1 -fi - -# ---------- Success ---------- -printf "\n" -printf "${GREEN}┌─────────────────────────────────────────────────────────────────┐${NC}\n" -printf "${GREEN}│${NC} ${GREEN}│${NC}\n" -printf "${GREEN}│${NC} ${BOLD}Installation Complete!${NC} ${GREEN}│${NC}\n" -printf "${GREEN}│${NC} ${GREEN}│${NC}\n" -printf "${GREEN}│${NC} Open ShipSec Studio in your browser: ${GREEN}│${NC}\n" -printf "${GREEN}│${NC} ${GREEN}│${NC}\n" -printf "${GREEN}│${NC} ${BOLD}http://localhost:8090${NC} ${GREEN}│${NC}\n" -printf "${GREEN}│${NC} ${GREEN}│${NC}\n" -printf "${GREEN}└─────────────────────────────────────────────────────────────────┘${NC}\n" -printf "\n" -info "Useful commands:" -printf "\n" -printf " just prod status - Check service status\n" -printf " just prod logs - View logs\n" -printf " just prod stop - Stop all services\n" -printf " just prod clean - Remove all data\n" -printf "\n" -info "Documentation: https://docs.shipsec.ai" -info "Need help? https://github.com/ShipSecAI/studio/issues" -printf "\n" - -exit 0 diff --git a/scripts/db-reset-instance.sh b/scripts/db-reset-instance.sh deleted file mode 100755 index a4fd64f9e..000000000 --- a/scripts/db-reset-instance.sh +++ /dev/null @@ -1,77 +0,0 @@ -#!/usr/bin/env bash -# Reset database for a specific instance -# Usage: ./scripts/db-reset-instance.sh [instance_number] - -set -euo pipefail - -INSTANCE=${1:-0} -COMPOSE_PROJECT_NAME="shipsec-infra" -DB_NAME="shipsec_instance_$INSTANCE" - -# Colors -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' - -log_info() { - echo -e "${BLUE}ℹ${NC} $*" -} - -log_success() { - echo -e "${GREEN}✅${NC} $*" -} - -log_error() { - echo -e "${RED}❌${NC} $*" -} - -log_info "Resetting database for instance $INSTANCE..." -echo "" - -# Find PostgreSQL container -POSTGRES_CONTAINER=$(docker compose -f docker/docker-compose.infra.yml \ - --project-name="$COMPOSE_PROJECT_NAME" \ - ps -q postgres 2>/dev/null || echo "") - -if [ -z "$POSTGRES_CONTAINER" ]; then - log_error "PostgreSQL container not found for instance $INSTANCE" - log_error "Is the instance running? Try: just dev $INSTANCE start" - exit 1 -fi - -log_info "Found PostgreSQL container: $POSTGRES_CONTAINER" - -# Drop and recreate database -log_info "Dropping database $DB_NAME..." -docker exec "$POSTGRES_CONTAINER" \ - psql -v ON_ERROR_STOP=1 -U shipsec -d postgres \ - -c "DROP DATABASE IF EXISTS \"$DB_NAME\";" || true - -log_info "Creating database $DB_NAME..." -docker exec "$POSTGRES_CONTAINER" \ - psql -v ON_ERROR_STOP=1 -U shipsec -d postgres \ - -c "CREATE DATABASE \"$DB_NAME\" OWNER shipsec;" - -docker exec "$POSTGRES_CONTAINER" \ - psql -v ON_ERROR_STOP=1 -U shipsec -d postgres \ - -c "GRANT ALL PRIVILEGES ON DATABASE \"$DB_NAME\" TO shipsec;" - -# Run migrations -log_info "Running migrations for instance $INSTANCE..." -export SHIPSEC_INSTANCE="$INSTANCE" -export DATABASE_URL="postgresql://shipsec:shipsec@localhost:5433/$DB_NAME" - -if bun --cwd backend run migration:push > /dev/null 2>&1; then - log_success "Migrations completed" -else - log_error "Migrations failed" - log_error "Check backend logs: just dev $INSTANCE logs" - exit 1 -fi - -echo "" -log_success "Database reset for instance $INSTANCE" -log_info "Database: $DB_NAME" -log_info "Connection: postgresql://shipsec:shipsec@localhost:5433/$DB_NAME" diff --git a/scripts/instance-clean.sh b/scripts/instance-clean.sh deleted file mode 100755 index 927353cd6..000000000 --- a/scripts/instance-clean.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env bash -# Clean shared infra resources for a specific instance. -# - Drop/recreate instance DB and re-run migrations (reset) -# - Delete Temporal namespace (best-effort) -# - Delete instance-scoped Kafka topics (best-effort) -# -# Usage: ./scripts/instance-clean.sh [instance_number] - -set -euo pipefail - -INSTANCE="${1:-0}" -INFRA_PROJECT_NAME="shipsec-infra" -DB_NAME="shipsec_instance_${INSTANCE}" -NAMESPACE="shipsec-dev-${INSTANCE}" -TEMPORAL_ADDRESS="127.0.0.1:7233" - -BLUE='\033[0;34m' -GREEN='\033[0;32m' -RED='\033[0;31m' -NC='\033[0m' - -log_info() { echo -e "${BLUE}ℹ${NC} $*"; } -log_success() { echo -e "${GREEN}✅${NC} $*"; } -log_error() { echo -e "${RED}❌${NC} $*"; } - -POSTGRES_CONTAINER="$( - docker compose -f docker/docker-compose.infra.yml --project-name="$INFRA_PROJECT_NAME" ps -q postgres 2>/dev/null || true -)" - -if [ -z "$POSTGRES_CONTAINER" ]; then - log_error "Postgres container not found (infra project: $INFRA_PROJECT_NAME). Is infra running?" - exit 1 -fi - -log_info "Resetting database for instance $INSTANCE..." -if ! ./scripts/db-reset-instance.sh "$INSTANCE" >/dev/null; then - log_error "Failed to reset database for instance $INSTANCE" - exit 1 -fi -log_success "Database reset complete" - -if command -v temporal >/dev/null 2>&1; then - log_info "Deleting Temporal namespace (best-effort): $NAMESPACE" - temporal operator namespace delete --address "$TEMPORAL_ADDRESS" --namespace "$NAMESPACE" --yes >/dev/null 2>&1 || true -fi - -REDPANDA_CONTAINER="$( - docker compose -f docker/docker-compose.infra.yml --project-name="$INFRA_PROJECT_NAME" ps -q redpanda 2>/dev/null || true -)" -if [ -n "$REDPANDA_CONTAINER" ]; then - log_info "Deleting Kafka topics for instance $INSTANCE (best-effort)..." - for base in telemetry.logs telemetry.events telemetry.agent-trace telemetry.node-io; do - topic="${base}.instance-${INSTANCE}" - docker exec "$REDPANDA_CONTAINER" rpk topic delete "$topic" --brokers redpanda:9092 >/dev/null 2>&1 || true - done -fi - -log_success "Instance $INSTANCE infra state cleaned" From 98d0d3eac9e747401498b6e74f760d43ebbbd352 Mon Sep 17 00:00:00 2001 From: betterclever Date: Thu, 12 Feb 2026 02:31:53 +0400 Subject: [PATCH 002/690] feat(worker): add runtime Job execution engine replacing DIND --- bun.lock | 129 ++-- packages/component-sdk/src/runner.ts | 26 + worker/package.json | 2 +- worker/src/temporal/workers/dev.worker.ts | 16 +- worker/src/utils/index.ts | 7 +- worker/src/utils/isolated-volume.ts | 21 + worker/src/utils/k8s-runner.ts | 692 ++++++++++++++++++++++ worker/src/utils/k8s-volume.ts | 235 ++++++++ 8 files changed, 1066 insertions(+), 62 deletions(-) create mode 100644 worker/src/utils/k8s-runner.ts create mode 100644 worker/src/utils/k8s-volume.ts diff --git a/bun.lock b/bun.lock index abaf42066..622d93467 100644 --- a/bun.lock +++ b/bun.lock @@ -10,15 +10,12 @@ "chalk": "^5.6.2", "chalk-animation": "^2.0.3", "long": "^5.3.2", - "zod": "^4.3.6", }, "devDependencies": { "@ai-sdk/mcp": "^1.0.13", - "@ai-sdk/openai": "^3.0.25", "@modelcontextprotocol/sdk": "^1.25.3", "@types/bun": "^1.3.6", "@types/node": "^24.10.9", - "ai": "^6.0.49", "bun-types": "^1.3.6", "husky": "^9.1.7", "lint-staged": "^16.2.7", @@ -36,15 +33,12 @@ "@clerk/backend": "^2.29.5", "@clerk/types": "^4.101.13", "@grpc/grpc-js": "^1.14.3", - "@nest-lab/throttler-storage-redis": "^1.1.0", "@nestjs/common": "^10.4.22", "@nestjs/config": "^3.3.0", "@nestjs/core": "^10.4.22", "@nestjs/microservices": "^11.1.13", "@nestjs/platform-express": "^10.4.22", "@nestjs/swagger": "^11.2.5", - "@nestjs/throttler": "^6.5.0", - "@opensearch-project/opensearch": "^3.5.1", "@shipsec/backend-client": "workspace:*", "@shipsec/component-sdk": "workspace:*", "@shipsec/shared": "workspace:*", @@ -60,7 +54,6 @@ "bcryptjs": "^3.0.3", "class-transformer": "^0.5.1", "class-validator": "^0.14.3", - "cookie-parser": "^1.4.7", "date-fns": "^4.1.0", "dotenv": "^17.2.3", "drizzle-orm": "^0.44.7", @@ -82,7 +75,6 @@ "@eslint/js": "^9.39.2", "@nestjs/testing": "^10.4.22", "@types/bcryptjs": "^3.0.0", - "@types/cookie-parser": "^1.4.10", "@types/express-serve-static-core": "^4.19.8", "@types/har-format": "^1.2.16", "@types/multer": "^2.0.0", @@ -92,7 +84,6 @@ "@typescript-eslint/eslint-plugin": "^8.53.1", "@typescript-eslint/parser": "^8.53.1", "bun-types": "^1.3.6", - "cookie-parser": "^1.4.7", "drizzle-kit": "^0.31.8", "eslint": "^9.39.2", "eslint-config-prettier": "^10.1.8", @@ -129,17 +120,13 @@ "@radix-ui/react-tooltip": "^1.2.8", "@shipsec/backend-client": "workspace:*", "@shipsec/shared": "workspace:*", - "@tanstack/react-query": "^5.90.21", - "@tanstack/react-query-devtools": "^5.91.3", "@uiw/react-markdown-preview": "^5.1.5", "ai": "^5.0.76", "ansi_up": "^6.0.6", "autoprefixer": "^10.4.21", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", - "date-fns": "^4.1.0", "dompurify": "^3.2.4", - "html-to-image": "1.11.11", "lucide-react": "^0.544.0", "markdown-it": "^14.1.0", "markdown-it-html5-embed": "^1.0.0", @@ -149,7 +136,6 @@ "postcss": "^8.5.6", "posthog-js": "^1.288.0", "react": "^19.2.0", - "react-day-picker": "^9.13.2", "react-dom": "^19.2.0", "react-router-dom": "^7.9.3", "reactflow": "^11.11.4", @@ -209,7 +195,6 @@ "name": "@shipsec/component-sdk", "version": "0.1.0", "dependencies": { - "@shipsec/shared": "workspace:*", "zod": "^4.3.6", }, "devDependencies": { @@ -267,9 +252,9 @@ "@aws-sdk/client-s3": "^3.975.0", "@googleapis/admin": "^29.0.0", "@grpc/grpc-js": "^1.14.3", + "@kubernetes/client-node": "^1.4.0", "@modelcontextprotocol/sdk": "^1.25.1", "@okta/okta-sdk-nodejs": "^7.3.0", - "@opensearch-project/opensearch": "^3.5.1", "@shipsec/component-sdk": "*", "@shipsec/contracts": "*", "@shipsec/shared": "*", @@ -491,8 +476,6 @@ "@csstools/css-tokenizer": ["@csstools/css-tokenizer@3.0.4", "", {}, "sha512-Vd/9EVDiu6PPJt9yAh6roZP6El1xHrdvIVGjyBsHR0RYwNHgL7FJPyIIW4fANJNG6FtyZfvlRPpFI4ZM/lubvw=="], - "@date-fns/tz": ["@date-fns/tz@1.4.1", "", {}, "sha512-P5LUNhtbj6YfI3iJjw5EL9eUAG6OitD0W3fWQcpQjDRc/QIsL0tRNuO1PcDvPccWL1fSTXXdE1ds+l95DV/OFA=="], - "@dnd-kit/accessibility": ["@dnd-kit/accessibility@3.1.1", "", { "dependencies": { "tslib": "^2.0.0" }, "peerDependencies": { "react": ">=16.8.0" } }, "sha512-2P+YgaXF+gRsIihwwY1gCsQSYnu9Zyj2py8kY5fFvUM1qm2WA2u639R6YNVfU4GWr+ZM5mqEsfHZZLoRONbemw=="], "@dnd-kit/core": ["@dnd-kit/core@6.3.1", "", { "dependencies": { "@dnd-kit/accessibility": "^3.1.1", "@dnd-kit/utilities": "^3.2.2", "tslib": "^2.0.0" }, "peerDependencies": { "react": ">=16.8.0", "react-dom": ">=16.8.0" } }, "sha512-xkGBRQQab4RLwgXxoqETICr6S5JlogafbhNsidmrkVv2YRs5MLwpjoF2qpiGjQt8S9AoxtIV603s0GIUpY5eYQ=="], @@ -621,6 +604,10 @@ "@js-sdsl/ordered-map": ["@js-sdsl/ordered-map@4.4.2", "", {}, "sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw=="], + "@jsep-plugin/assignment": ["@jsep-plugin/assignment@1.3.0", "", { "peerDependencies": { "jsep": "^0.4.0||^1.0.0" } }, "sha512-VVgV+CXrhbMI3aSusQyclHkenWSAm95WaiKrMxRFam3JSUiIaQjoMIw2sEs/OX4XifnqeQUN4DYbJjlA8EfktQ=="], + + "@jsep-plugin/regex": ["@jsep-plugin/regex@1.0.4", "", { "peerDependencies": { "jsep": "^0.4.0||^1.0.0" } }, "sha512-q7qL4Mgjs1vByCaTnDFcBnV9HS7GVPJX5vyVoCgZHNSC9rjwIlmbXG5sUuorR5ndfHAIlJ8pVStxvjXHbNvtUg=="], + "@jsonjoy.com/base64": ["@jsonjoy.com/base64@1.1.2", "", { "peerDependencies": { "tslib": "2" } }, "sha512-q6XAnWQDIMA3+FTiOYajoYqySkO+JSat0ytXGSuRdq9uXE7o92gzuQwQM14xaCRlBLGq3v5miDGC4vkVTn54xA=="], "@jsonjoy.com/buffers": ["@jsonjoy.com/buffers@17.65.0", "", { "peerDependencies": { "tslib": "2" } }, "sha512-eBrIXd0/Ld3p9lpDDlMaMn6IEfWqtHMD+z61u0JrIiPzsV1r7m6xDZFRxJyvIFTEO+SWdYF9EiQbXZGd8BzPfA=="], @@ -649,6 +636,8 @@ "@jsonjoy.com/util": ["@jsonjoy.com/util@1.9.0", "", { "dependencies": { "@jsonjoy.com/buffers": "^1.0.0", "@jsonjoy.com/codegen": "^1.0.0" }, "peerDependencies": { "tslib": "2" } }, "sha512-pLuQo+VPRnN8hfPqUTLTHk126wuYdXVxE6aDmjSeV4NCAgyxWbiOIeNJVtID3h1Vzpoi9m4jXezf73I6LgabgQ=="], + "@kubernetes/client-node": ["@kubernetes/client-node@1.4.0", "", { "dependencies": { "@types/js-yaml": "^4.0.1", "@types/node": "^24.0.0", "@types/node-fetch": "^2.6.13", "@types/stream-buffers": "^3.0.3", "form-data": "^4.0.0", "hpagent": "^1.2.0", "isomorphic-ws": "^5.0.0", "js-yaml": "^4.1.0", "jsonpath-plus": "^10.3.0", "node-fetch": "^2.7.0", "openid-client": "^6.1.3", "rfc4648": "^1.3.0", "socks-proxy-agent": "^8.0.4", "stream-buffers": "^3.0.2", "tar-fs": "^3.0.9", "ws": "^8.18.2" } }, "sha512-Zge3YvF7DJi264dU1b3wb/GmzR99JhUpqTvp+VGHfwZT+g7EOOYNScDJNZwXy9cszyIGPIs0VHr+kk8e95qqrA=="], + "@lukeed/csprng": ["@lukeed/csprng@1.1.0", "", {}, "sha512-Z7C/xXCiGWsg0KuKsHTKJxbWhpI3Vs5GwLfOean7MGyVFGqdRgBbAjOCh6u4bbjPc/8MJ2pZmK/0DLdCbivLDA=="], "@microsoft/tsdoc": ["@microsoft/tsdoc@0.16.0", "", {}, "sha512-xgAyonlVVS+q7Vc7qLW0UrJU7rSFcETRWsqdXZtjzRU8dF+6CkozTK4V4y1LwOX7j8r/vHphjDeMeGI4tNGeGA=="], @@ -659,8 +648,6 @@ "@monaco-editor/react": ["@monaco-editor/react@4.7.0", "", { "dependencies": { "@monaco-editor/loader": "^1.5.0" }, "peerDependencies": { "monaco-editor": ">= 0.25.0 < 1", "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, "sha512-cyzXQCtO47ydzxpQtCGSQGOC8Gk3ZUeBXFAxD+CWXYFo5OqZyZUonFl0DwUlTyAfRHntBfw2p3w4s9R6oe1eCA=="], - "@nest-lab/throttler-storage-redis": ["@nest-lab/throttler-storage-redis@1.2.0", "", { "dependencies": { "tslib": "^2.3.0" }, "peerDependencies": { "@nestjs/common": "^7.0.0 || ^8.0.0 || ^9.0.0 || ^10.0.0 || ^11.0.0", "@nestjs/core": "^7.0.0 || ^8.0.0 || ^9.0.0 || ^10.0.0 || ^11.0.0", "@nestjs/throttler": ">=6.0.0", "ioredis": ">=5.0.0", "reflect-metadata": "^0.2.1" } }, "sha512-tMkUyo68NCKTR+zILk+EC35SMYBtDPZY2mCj7ZaCietWGVTnuP4zwq9ERYfvU6kJv6h8teNZrC6MJCmY6/dljw=="], - "@nestjs/common": ["@nestjs/common@10.4.22", "", { "dependencies": { "file-type": "20.4.1", "iterare": "1.2.1", "tslib": "2.8.1", "uid": "2.0.2" }, "peerDependencies": { "class-transformer": "*", "class-validator": "*", "reflect-metadata": "^0.1.12 || ^0.2.0", "rxjs": "^7.1.0" }, "optionalPeers": ["class-transformer", "class-validator"] }, "sha512-fxJ4v85nDHaqT1PmfNCQ37b/jcv2OojtXTaK1P2uAXhzLf9qq6WNUOFvxBrV4fhQek1EQoT1o9oj5xAZmv3NRw=="], "@nestjs/config": ["@nestjs/config@3.3.0", "", { "dependencies": { "dotenv": "16.4.5", "dotenv-expand": "10.0.0", "lodash": "4.17.21" }, "peerDependencies": { "@nestjs/common": "^8.0.0 || ^9.0.0 || ^10.0.0", "rxjs": "^7.1.0" } }, "sha512-pdGTp8m9d0ZCrjTpjkUbZx6gyf2IKf+7zlkrPNMsJzYZ4bFRRTpXrnj+556/5uiI6AfL5mMrJc2u7dB6bvM+VA=="], @@ -677,8 +664,6 @@ "@nestjs/testing": ["@nestjs/testing@10.4.22", "", { "dependencies": { "tslib": "2.8.1" }, "peerDependencies": { "@nestjs/common": "^10.0.0", "@nestjs/core": "^10.0.0", "@nestjs/microservices": "^10.0.0", "@nestjs/platform-express": "^10.0.0" }, "optionalPeers": ["@nestjs/microservices", "@nestjs/platform-express"] }, "sha512-HO9aPus3bAedAC+jKVAA8jTdaj4fs5M9fing4giHrcYV2txe9CvC1l1WAjwQ9RDhEHdugjY4y+FZA/U/YqPZrA=="], - "@nestjs/throttler": ["@nestjs/throttler@6.5.0", "", { "peerDependencies": { "@nestjs/common": "^7.0.0 || ^8.0.0 || ^9.0.0 || ^10.0.0 || ^11.0.0", "@nestjs/core": "^7.0.0 || ^8.0.0 || ^9.0.0 || ^10.0.0 || ^11.0.0", "reflect-metadata": "^0.1.13 || ^0.2.0" } }, "sha512-9j0ZRfH0QE1qyrj9JjIRDz5gQLPqq9yVC2nHsrosDVAfI5HHw08/aUAWx9DZLSdQf4HDkmhTTEGLrRFHENvchQ=="], - "@noble/hashes": ["@noble/hashes@1.8.0", "", {}, "sha512-jCs9ldd7NwzpgXDIf6P3+NrHh9/sD6CQdxHyjQI+h/6rDNo88ypBxxz45UDuZHz9r3tNz7N/VInSVoVdtXEI4A=="], "@nodelib/fs.scandir": ["@nodelib/fs.scandir@2.1.5", "", { "dependencies": { "@nodelib/fs.stat": "2.0.5", "run-parallel": "^1.1.9" } }, "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g=="], @@ -691,8 +676,6 @@ "@okta/okta-sdk-nodejs": ["@okta/okta-sdk-nodejs@7.3.0", "", { "dependencies": { "@types/node-forge": "^1.3.1", "deep-copy": "^1.4.2", "eckles": "^1.4.1", "form-data": "^4.0.4", "https-proxy-agent": "^5.0.0", "js-yaml": "^4.1.0", "lodash": "^4.17.20", "njwt": "^2.0.1", "node-fetch": "^2.6.7", "node-jose": "^2.2.0", "parse-link-header": "^2.0.0", "rasha": "^1.2.5", "safe-flat": "^2.0.2", "url-parse": "^1.5.10", "uuid": "^11.1.0" } }, "sha512-6J3VV+8fBOqIXDqb3t2sBeXj1WOEZL6wP2AcGRzvMRMb2WL7JKR6ZDrt/1Kk7j4seXCKMpZrHsPYYdfRXwkSKQ=="], - "@opensearch-project/opensearch": ["@opensearch-project/opensearch@3.5.1", "", { "dependencies": { "aws4": "^1.11.0", "debug": "^4.3.1", "hpagent": "^1.2.0", "json11": "^2.0.0", "ms": "^2.1.3", "secure-json-parse": "^2.4.0" } }, "sha512-6bf+HcuERzAtHZxrm6phjref54ABse39BpkDie/YO3AUFMCBrb3SK5okKSdT5n3+nDRuEEQLhQCl0RQV3s1qpA=="], - "@opentelemetry/api": ["@opentelemetry/api@1.9.0", "", {}, "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg=="], "@opentelemetry/api-logs": ["@opentelemetry/api-logs@0.208.0", "", { "dependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-CjruKY9V6NMssL/T1kAFgzosF1v9o6oeN+aX5JB/C/xPNtmgIJqcXHG7fA82Ou1zCpWGl4lROQUKwUNE1pMCyg=="], @@ -1061,14 +1044,6 @@ "@tailwindcss/typography": ["@tailwindcss/typography@0.5.19", "", { "dependencies": { "postcss-selector-parser": "6.0.10" }, "peerDependencies": { "tailwindcss": ">=3.0.0 || insiders || >=4.0.0-alpha.20 || >=4.0.0-beta.1" } }, "sha512-w31dd8HOx3k9vPtcQh5QHP9GwKcgbMp87j58qi6xgiBnFFtKEAgCWnDw4qUT8aHwkCp8bKvb/KGKWWHedP0AAg=="], - "@tanstack/query-core": ["@tanstack/query-core@5.90.20", "", {}, "sha512-OMD2HLpNouXEfZJWcKeVKUgQ5n+n3A2JFmBaScpNDUqSrQSjiveC7dKMe53uJUg1nDG16ttFPz2xfilz6i2uVg=="], - - "@tanstack/query-devtools": ["@tanstack/query-devtools@5.93.0", "", {}, "sha512-+kpsx1NQnOFTZsw6HAFCW3HkKg0+2cepGtAWXjiiSOJJ1CtQpt72EE2nyZb+AjAbLRPoeRmPJ8MtQd8r8gsPdg=="], - - "@tanstack/react-query": ["@tanstack/react-query@5.90.21", "", { "dependencies": { "@tanstack/query-core": "5.90.20" }, "peerDependencies": { "react": "^18 || ^19" } }, "sha512-0Lu6y5t+tvlTJMTO7oh5NSpJfpg/5D41LlThfepTixPYkJ0sE2Jj0m0f6yYqujBwIXlId87e234+MxG3D3g7kg=="], - - "@tanstack/react-query-devtools": ["@tanstack/react-query-devtools@5.91.3", "", { "dependencies": { "@tanstack/query-devtools": "5.93.0" }, "peerDependencies": { "@tanstack/react-query": "^5.90.20", "react": "^18 || ^19" } }, "sha512-nlahjMtd/J1h7IzOOfqeyDh5LNfG0eULwlltPEonYy0QL+nqrBB+nyzJfULV+moL7sZyxc2sHdNJki+vLA9BSA=="], - "@temporalio/activity": ["@temporalio/activity@1.14.1", "", { "dependencies": { "@temporalio/client": "1.14.1", "@temporalio/common": "1.14.1", "abort-controller": "^3.0.0" } }, "sha512-wG2fTNgomhcKOzPY7mqhKqe8scawm4BvUYdgX1HJouHmVNRgtZurf2xQWJZQOTxWrsXfdoYqzohZLzxlNtcC5A=="], "@temporalio/client": ["@temporalio/client@1.14.1", "", { "dependencies": { "@grpc/grpc-js": "^1.12.4", "@temporalio/common": "1.14.1", "@temporalio/proto": "1.14.1", "abort-controller": "^3.0.0", "long": "^5.2.3", "uuid": "^11.1.0" } }, "sha512-AfWSA0LYzBvDLFiFgrPWqTGGq1NGnF3d4xKnxf0PGxSmv5SLb/aqQ9lzHg4DJ5UNkHO4M/NwzdxzzoaR1J5F8Q=="], @@ -1117,8 +1092,6 @@ "@types/connect": ["@types/connect@3.4.38", "", { "dependencies": { "@types/node": "*" } }, "sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug=="], - "@types/cookie-parser": ["@types/cookie-parser@1.4.10", "", { "peerDependencies": { "@types/express": "*" } }, "sha512-B4xqkqfZ8Wek+rCOeRxsjMS9OgvzebEzzLYw7NHYuvzb7IdxOkI0ZHGgeEBX4PUM7QGVvNSK60T3OvWj3YfBRg=="], - "@types/cookiejar": ["@types/cookiejar@2.1.5", "", {}, "sha512-he+DHOWReW0nghN24E1WUqM0efK4kI9oTqDm6XmK8ZPe2djZ90BSNdGnIyCLzCPw7/pogPlGbzI2wHGGmi4O/Q=="], "@types/d3": ["@types/d3@7.4.3", "", { "dependencies": { "@types/d3-array": "*", "@types/d3-axis": "*", "@types/d3-brush": "*", "@types/d3-chord": "*", "@types/d3-color": "*", "@types/d3-contour": "*", "@types/d3-delaunay": "*", "@types/d3-dispatch": "*", "@types/d3-drag": "*", "@types/d3-dsv": "*", "@types/d3-ease": "*", "@types/d3-fetch": "*", "@types/d3-force": "*", "@types/d3-format": "*", "@types/d3-geo": "*", "@types/d3-hierarchy": "*", "@types/d3-interpolate": "*", "@types/d3-path": "*", "@types/d3-polygon": "*", "@types/d3-quadtree": "*", "@types/d3-random": "*", "@types/d3-scale": "*", "@types/d3-scale-chromatic": "*", "@types/d3-selection": "*", "@types/d3-shape": "*", "@types/d3-time": "*", "@types/d3-time-format": "*", "@types/d3-timer": "*", "@types/d3-transition": "*", "@types/d3-zoom": "*" } }, "sha512-lZXZ9ckh5R8uiFVt8ogUNf+pIrK4EsWrx2Np75WvF/eTpJ0FMHNhjXk8CKEx/+gpHbNQyJWehbFaTvqmHWB3ww=="], @@ -1233,6 +1206,8 @@ "@types/node": ["@types/node@24.10.9", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-ne4A0IpG3+2ETuREInjPNhUGis1SFjv1d5asp8MzEAGtOZeTeHVDOYqOgqfhvseqg/iXty2hjBf1zAOb7RNiNw=="], + "@types/node-fetch": ["@types/node-fetch@2.6.13", "", { "dependencies": { "@types/node": "*", "form-data": "^4.0.4" } }, "sha512-QGpRVpzSaUs30JBSGPjOg4Uveu384erbHBoT1zeONvyCfwQxIkUshLAOqN/k9EjGviPRmWTTe6aH2qySWKTVSw=="], + "@types/node-forge": ["@types/node-forge@1.3.14", "", { "dependencies": { "@types/node": "*" } }, "sha512-mhVF2BnD4BO+jtOp7z1CdzaK4mbuK0LLQYAvdOLqHTavxFNq4zA1EmYkpnFjP8HOUzedfQkRnp0E2ulSAYSzAw=="], "@types/normalize-package-data": ["@types/normalize-package-data@2.4.4", "", {}, "sha512-37i+OaWTh9qeK4LSHPsyRC7NahnGotNuZvjLSgcPzblpHB3rrCJxAOgI5gCdKm7coonsaX1Of0ILiTcnZjbfxA=="], @@ -1255,6 +1230,8 @@ "@types/serve-static": ["@types/serve-static@2.2.0", "", { "dependencies": { "@types/http-errors": "*", "@types/node": "*" } }, "sha512-8mam4H1NHLtu7nmtalF7eyBH14QyOASmcxHhSfEoRyr0nP/YdoesEtU+uSRvMe96TW/HPTtkoKqQLl53N7UXMQ=="], + "@types/stream-buffers": ["@types/stream-buffers@3.0.8", "", { "dependencies": { "@types/node": "*" } }, "sha512-J+7VaHKNvlNPJPEJXX/fKa9DZtR/xPMwuIbe+yNOwp1YB+ApUOBv2aUpEoBJEi8nJgbgs1x8e73ttg0r1rSUdw=="], + "@types/superagent": ["@types/superagent@8.1.9", "", { "dependencies": { "@types/cookiejar": "^2.1.5", "@types/methods": "^1.1.4", "@types/node": "*", "form-data": "^4.0.0" } }, "sha512-pTVjI73witn+9ILmoJdajHGW2jkSaOzhiFYF1Rd3EQ94kymLqB9PjD9ISg7WaALC7+dCHT0FGe9T2LktLq/3GQ=="], "@types/supertest": ["@types/supertest@2.0.16", "", { "dependencies": { "@types/superagent": "*" } }, "sha512-6c2ogktZ06tr2ENoZivgm7YnprnhYE4ZoXGMY+oA7IuAf17M8FWvujXZGmxLv8y0PTyts4x5A+erSwVUFA8XSg=="], @@ -1423,12 +1400,24 @@ "available-typed-arrays": ["available-typed-arrays@1.0.7", "", { "dependencies": { "possible-typed-array-names": "^1.0.0" } }, "sha512-wvUjBtSGN7+7SjNpq/9M2Tg350UZD3q62IFZLbRAR1bSMlCo1ZaeW+BJ+D090e4hIIZLBcTDWe4Mh4jvUDajzQ=="], - "aws4": ["aws4@1.13.2", "", {}, "sha512-lHe62zvbTB5eEABUVi/AwVh0ZKY9rMMDhmm+eeyuuUQbQ3+J+fONVQOZyj+DdrvD4BY33uYniyRJ4UJIaSKAfw=="], + "b4a": ["b4a@1.7.3", "", { "peerDependencies": { "react-native-b4a": "*" }, "optionalPeers": ["react-native-b4a"] }, "sha512-5Q2mfq2WfGuFp3uS//0s6baOJLMoVduPYVeNmDYxu5OUA1/cBfvr2RIS7vi62LdNj/urk1hfmj867I3qt6uZ7Q=="], "bail": ["bail@2.0.2", "", {}, "sha512-0xO6mYd7JB2YesxDKplafRpsiOzPt9V02ddPCLbY1xYGPOX24NTyN50qnUxgCPcSoYMhKpAuBTjQoRZCAkUDRw=="], "balanced-match": ["balanced-match@1.0.2", "", {}, "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw=="], + "bare-events": ["bare-events@2.8.2", "", { "peerDependencies": { "bare-abort-controller": "*" }, "optionalPeers": ["bare-abort-controller"] }, "sha512-riJjyv1/mHLIPX4RwiK+oW9/4c3TEUeORHKefKAKnZ5kyslbN+HXowtbaVEqt4IMUB7OXlfixcs6gsFeo/jhiQ=="], + + "bare-fs": ["bare-fs@4.5.3", "", { "dependencies": { "bare-events": "^2.5.4", "bare-path": "^3.0.0", "bare-stream": "^2.6.4", "bare-url": "^2.2.2", "fast-fifo": "^1.3.2" }, "peerDependencies": { "bare-buffer": "*" }, "optionalPeers": ["bare-buffer"] }, "sha512-9+kwVx8QYvt3hPWnmb19tPnh38c6Nihz8Lx3t0g9+4GoIf3/fTgYwM4Z6NxgI+B9elLQA7mLE9PpqcWtOMRDiQ=="], + + "bare-os": ["bare-os@3.6.2", "", {}, "sha512-T+V1+1srU2qYNBmJCXZkUY5vQ0B4FSlL3QDROnKQYOqeiQR8UbjNHlPa+TIbM4cuidiN9GaTaOZgSEgsvPbh5A=="], + + "bare-path": ["bare-path@3.0.0", "", { "dependencies": { "bare-os": "^3.0.1" } }, "sha512-tyfW2cQcB5NN8Saijrhqn0Zh7AnFNsnczRcuWODH0eYAXBsJ5gVxAUuNr7tsHSC6IZ77cA0SitzT+s47kot8Mw=="], + + "bare-stream": ["bare-stream@2.7.0", "", { "dependencies": { "streamx": "^2.21.0" }, "peerDependencies": { "bare-buffer": "*", "bare-events": "*" }, "optionalPeers": ["bare-buffer", "bare-events"] }, "sha512-oyXQNicV1y8nc2aKffH+BUHFRXmx6VrPzlnaEvMhram0nPBrKcEdcyBg5r08D0i8VxngHFAiVyn1QKXpSG0B8A=="], + + "bare-url": ["bare-url@2.3.2", "", { "dependencies": { "bare-path": "^3.0.0" } }, "sha512-ZMq4gd9ngV5aTMa5p9+UfY0b3skwhHELaDkhEHetMdX0LRkW9kzaym4oo/Eh+Ghm0CCDuMTsRIGM/ytUc1ZYmw=="], + "base64-js": ["base64-js@1.5.1", "", {}, "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA=="], "base64url": ["base64url@3.0.1", "", {}, "sha512-ir1UPr3dkwexU7FdV8qBBbNDRUhMmIekYMFZfi+C/sLNnRESKPl23nB9b2pltqfOQNnGzsDdId90AEtG5tCx4A=="], @@ -1573,9 +1562,7 @@ "cookie": ["cookie@0.7.2", "", {}, "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w=="], - "cookie-parser": ["cookie-parser@1.4.7", "", { "dependencies": { "cookie": "0.7.2", "cookie-signature": "1.0.6" } }, "sha512-nGUvgXnotP3BsjiLX2ypbQnWoGUPIIfHQNZkkC668ntrzGWEZVW70HDEB1qnNGMicPje6EttlIgzo51YSwNQGw=="], - - "cookie-signature": ["cookie-signature@1.0.6", "", {}, "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ=="], + "cookie-signature": ["cookie-signature@1.2.2", "", {}, "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg=="], "cookiejar": ["cookiejar@2.1.4", "", {}, "sha512-LDx6oHrK+PhzLKJU9j5S7/Y3jM/mUHvD/DeI1WQmJn652iPC5Y4TBzC9l+5OMOXlyTTA+SmVUPm0HQUwpD5Jqw=="], @@ -1631,8 +1618,6 @@ "date-fns": ["date-fns@4.1.0", "", {}, "sha512-Ukq0owbQXxa/U3EGtsdVBkR1w7KOQ5gIBqdH2hkvknzZPYvBxb/aa6E8L7tmjFtkwZBu3UXBbjIgPo/Ez4xaNg=="], - "date-fns-jalali": ["date-fns-jalali@4.1.0-0", "", {}, "sha512-hTIP/z+t+qKwBDcmmsnmjWTduxCg+5KfdqWQvb2X/8C9+knYY6epN/pfxdDuyVlSVeFz0sM5eEfwIUQ70U4ckg=="], - "dayjs": ["dayjs@1.11.15", "", {}, "sha512-MC+DfnSWiM9APs7fpiurHGCoeIx0Gdl6QZBy+5lu8MbYKN5FZEXqOgrundfibdfhGZ15o9hzmZ2xJjZnbvgKXQ=="], "debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="], @@ -1711,6 +1696,8 @@ "encodeurl": ["encodeurl@2.0.0", "", {}, "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg=="], + "end-of-stream": ["end-of-stream@1.4.5", "", { "dependencies": { "once": "^1.4.0" } }, "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg=="], + "enhanced-resolve": ["enhanced-resolve@5.18.4", "", { "dependencies": { "graceful-fs": "^4.2.4", "tapable": "^2.2.0" } }, "sha512-LgQMM4WXU3QI+SYgEc2liRgznaD5ojbmY3sb8LxyguVkIg5FxdpTkvk72te2R38/TGKxH634oLxXRGY6d7AP+Q=="], "enquirer": ["enquirer@2.3.6", "", { "dependencies": { "ansi-colors": "^4.1.1" } }, "sha512-yjNnPr315/FjS4zIsUxYguYUPP2e1NK4d7E7ZOLiyYCcbFBiTMyID+2wvm2w6+pZ/odMA7cRkjhsPbltwBOrLg=="], @@ -1793,6 +1780,8 @@ "events": ["events@3.3.0", "", {}, "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q=="], + "events-universal": ["events-universal@1.0.1", "", { "dependencies": { "bare-events": "^2.7.0" } }, "sha512-LUd5euvbMLpwOF8m6ivPCbhQeSiYVNb8Vs0fQ8QjXo0JTkEHpz8pxdQf0gStltaPpw0Cca8b39KxvK9cfKRiAw=="], + "eventsource": ["eventsource@3.0.7", "", { "dependencies": { "eventsource-parser": "^3.0.1" } }, "sha512-CRT1WTyuQoD771GW56XEZFQ/ZoSfWid1alKGDYMmkt2yl8UXrVR4pspqWNEcqKvVIzg6PAltWjxcSSPrboA4iA=="], "eventsource-parser": ["eventsource-parser@3.0.6", "", {}, "sha512-Vo1ab+QXPzZ4tCa8SwIHJFaSzy4R6SHf7BY79rFBDf0idraZWAkYrDjDj8uWaSm3S2TK+hJ7/t1CEmZ7jXw+pg=="], @@ -1809,6 +1798,8 @@ "fast-diff": ["fast-diff@1.3.0", "", {}, "sha512-VxPP4NqbUjj6MaAOafWeUn2cXWLcCtljklUtZf0Ind4XQ+QPtmA0b18zZy0jIQx+ExRVCR/ZQpBmik5lXshNsw=="], + "fast-fifo": ["fast-fifo@1.3.2", "", {}, "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ=="], + "fast-glob": ["fast-glob@3.3.3", "", { "dependencies": { "@nodelib/fs.stat": "^2.0.2", "@nodelib/fs.walk": "^1.2.3", "glob-parent": "^5.1.2", "merge2": "^1.3.0", "micromatch": "^4.0.8" } }, "sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg=="], "fast-json-patch": ["fast-json-patch@3.1.1", "", {}, "sha512-vf6IHUX2SBcA+5/+4883dsIjpBTqmfBjmYiWK1savxQmFk4JfBMLa7ynTYOs1Rolp/T1betJxHiGD3g1Mn8lUQ=="], @@ -1993,8 +1984,6 @@ "html-encoding-sniffer": ["html-encoding-sniffer@6.0.0", "", { "dependencies": { "@exodus/bytes": "^1.6.0" } }, "sha512-CV9TW3Y3f8/wT0BRFc1/KAVQ3TUHiXmaAb6VW9vtiMFf7SLoMd1PdAc4W3KFOFETBJUb90KatHqlsZMWV+R9Gg=="], - "html-to-image": ["html-to-image@1.11.11", "", {}, "sha512-9gux8QhvjRO/erSnDPv28noDZcPZmYE7e1vFsBLKLlRlKDSqNJYebj6Qz1TGd5lsRV+X+xYyjCKjuZdABinWjA=="], - "html-url-attributes": ["html-url-attributes@3.0.1", "", {}, "sha512-ol6UPyBWqsrO6EJySPz2O7ZSr856WDrEzM5zMqp+FJJLGMW35cLYmmZnl0vztAZxRUoNZJFTCohfjuIJ8I4QBQ=="], "html-void-elements": ["html-void-elements@3.0.0", "", {}, "sha512-bEqo66MRXsUGxWHV5IP0PUiAWwoEjba4VCzg0LjFJBpchPaTfyfCKTG6bc5F8ucKec3q5y6qOdGyYTSBEvhCrg=="], @@ -2113,6 +2102,8 @@ "isexe": ["isexe@2.0.0", "", {}, "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw=="], + "isomorphic-ws": ["isomorphic-ws@5.0.0", "", { "peerDependencies": { "ws": "*" } }, "sha512-muId7Zzn9ywDsyXgTIafTry2sV3nySZeUDe6YedVd1Hvuuep5AsIlqK+XefWpYTyJG5e503F2xIuT2lcU6rCSw=="], + "iterare": ["iterare@1.2.1", "", {}, "sha512-RKYVTCjAnRthyJes037NX/IiqeidgN1xc3j1RjFfECFp28A1GVwK9nA+i0rJPaHqSZwygLzRnFlzUuHFoWWy+Q=="], "iterator.prototype": ["iterator.prototype@1.1.5", "", { "dependencies": { "define-data-property": "^1.1.4", "es-object-atoms": "^1.0.0", "get-intrinsic": "^1.2.6", "get-proto": "^1.0.0", "has-symbols": "^1.1.0", "set-function-name": "^2.0.2" } }, "sha512-H0dkQoCa3b2VEeKQBOxFph+JAbcrQdE7KC0UkqwpLmv2EC4P41QXP+rqo9wYodACiG5/WM5s9oDApTU8utwj9g=="], @@ -2139,6 +2130,8 @@ "jsdom": ["jsdom@27.4.0", "", { "dependencies": { "@acemir/cssom": "^0.9.28", "@asamuzakjp/dom-selector": "^6.7.6", "@exodus/bytes": "^1.6.0", "cssstyle": "^5.3.4", "data-urls": "^6.0.0", "decimal.js": "^10.6.0", "html-encoding-sniffer": "^6.0.0", "http-proxy-agent": "^7.0.2", "https-proxy-agent": "^7.0.6", "is-potential-custom-element-name": "^1.0.1", "parse5": "^8.0.0", "saxes": "^6.0.0", "symbol-tree": "^3.2.4", "tough-cookie": "^6.0.0", "w3c-xmlserializer": "^5.0.0", "webidl-conversions": "^8.0.0", "whatwg-mimetype": "^4.0.0", "whatwg-url": "^15.1.0", "ws": "^8.18.3", "xml-name-validator": "^5.0.0" }, "peerDependencies": { "canvas": "^3.0.0" }, "optionalPeers": ["canvas"] }, "sha512-mjzqwWRD9Y1J1KUi7W97Gja1bwOOM5Ug0EZ6UDK3xS7j7mndrkwozHtSblfomlzyB4NepioNt+B2sOSzczVgtQ=="], + "jsep": ["jsep@1.4.0", "", {}, "sha512-B7qPcEVE3NVkmSJbaYxvv4cHkVW7DQsZz13pUMrfS8z8Q/BuShN+gcTXrUlPiGqM2/t/EEaI030bpxMqY8gMlw=="], + "jsesc": ["jsesc@3.1.0", "", { "bin": { "jsesc": "bin/jsesc" } }, "sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA=="], "json-bigint": ["json-bigint@1.0.0", "", { "dependencies": { "bignumber.js": "^9.0.0" } }, "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ=="], @@ -2157,10 +2150,10 @@ "json-stringify-safe": ["json-stringify-safe@5.0.1", "", {}, "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA=="], - "json11": ["json11@2.0.2", "", { "bin": { "json11": "dist/cli.mjs" } }, "sha512-HIrd50UPYmP6sqLuLbFVm75g16o0oZrVfxrsY0EEys22klz8mRoWlX9KAEDOSOR9Q34rcxsyC8oDveGrCz5uLQ=="], - "json5": ["json5@2.2.3", "", { "bin": { "json5": "lib/cli.js" } }, "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg=="], + "jsonpath-plus": ["jsonpath-plus@10.3.0", "", { "dependencies": { "@jsep-plugin/assignment": "^1.3.0", "@jsep-plugin/regex": "^1.0.4", "jsep": "^1.4.0" }, "bin": { "jsonpath": "bin/jsonpath-cli.js", "jsonpath-plus": "bin/jsonpath-cli.js" } }, "sha512-8TNmfeTCk2Le33A3vRRwtuworG/L5RrgMvdjhKZxvyShO+mBu2fP50OWUjRLNtvw344DdDarFh9buFAZs5ujeA=="], + "jsx-ast-utils": ["jsx-ast-utils@3.3.5", "", { "dependencies": { "array-includes": "^3.1.6", "array.prototype.flat": "^1.3.1", "object.assign": "^4.1.4", "object.values": "^1.1.6" } }, "sha512-ZZow9HBI5O6EPgSJLUb8n2NKgmVWTwCvHGwFuJlMjvLFqlGG6pjirPhtdsseaLZjSibD8eegzmYpUZwoIlj2cQ=="], "jwa": ["jwa@2.0.1", "", { "dependencies": { "buffer-equal-constant-time": "^1.0.1", "ecdsa-sig-formatter": "1.0.11", "safe-buffer": "^5.0.1" } }, "sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg=="], @@ -2371,7 +2364,7 @@ "mqtt-packet": ["mqtt-packet@9.0.2", "", { "dependencies": { "bl": "^6.0.8", "debug": "^4.3.4", "process-nextick-args": "^2.0.1" } }, "sha512-MvIY0B8/qjq7bKxdN1eD+nrljoeaai+qjLJgfRn3TiMuz0pamsIWY2bFODPZMSNmabsLANXsLl4EMoWvlaTZWA=="], - "ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], + "ms": ["ms@3.0.0-canary.1", "", {}, "sha512-kh8ARjh8rMN7Du2igDRO9QJnqCb2xYTJxyQYK7vJJS4TvLLmsbyhiKpSW+t+y26gyOyMd0riphX0GeWKU3ky5g=="], "multer": ["multer@2.0.2", "", { "dependencies": { "append-field": "^1.0.0", "busboy": "^1.6.0", "concat-stream": "^2.0.0", "mkdirp": "^0.5.6", "object-assign": "^4.1.1", "type-is": "^1.6.18", "xtend": "^4.0.2" } }, "sha512-u7f2xaZ/UG8oLXHvtF/oWTRvT44p9ecwBBqTwgJVq0+4BW1g8OW01TyMEGWBHbyMOYVHXslaut7qEQ1meATXgw=="], @@ -2421,6 +2414,8 @@ "number-allocator": ["number-allocator@1.0.14", "", { "dependencies": { "debug": "^4.3.1", "js-sdsl": "4.3.0" } }, "sha512-OrL44UTVAvkKdOdRQZIJpLkAdjXGTRda052sN4sO77bKEzYYqWKMBjQvrJFzqygI99gL6Z4u2xctPW1tB8ErvA=="], + "oauth4webapi": ["oauth4webapi@3.8.4", "", {}, "sha512-EKlVEgav8zH31IXxvhCqjEgQws6S9QmnmJyLXmeV5REf59g7VmqRVa5l/rhGWtUqGm2rLVTNwukn9hla5kJ2WQ=="], + "object-assign": ["object-assign@4.1.1", "", {}, "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg=="], "object-hash": ["object-hash@3.0.0", "", {}, "sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw=="], @@ -2449,6 +2444,8 @@ "openapi-typescript-helpers": ["openapi-typescript-helpers@0.0.15", "", {}, "sha512-opyTPaunsklCBpTK8JGef6mfPhLSnyy5a0IN9vKtx3+4aExf+KxEqYwIy3hqkedXIB97u357uLMJsOnm3GVjsw=="], + "openid-client": ["openid-client@6.8.2", "", { "dependencies": { "jose": "^6.1.3", "oauth4webapi": "^3.8.4" } }, "sha512-uOvTCndr4udZsKihJ68H9bUICrriHdUVJ6Az+4Ns6cW55rwM5h0bjVIzDz2SxgOI84LKjFyjOFvERLzdTUROGA=="], + "optionator": ["optionator@0.9.4", "", { "dependencies": { "deep-is": "^0.1.3", "fast-levenshtein": "^2.0.6", "levn": "^0.4.1", "prelude-ls": "^1.2.1", "type-check": "^0.4.0", "word-wrap": "^1.2.5" } }, "sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g=="], "own-keys": ["own-keys@1.0.1", "", { "dependencies": { "get-intrinsic": "^1.2.6", "object-keys": "^1.1.1", "safe-push-apply": "^1.0.0" } }, "sha512-qFOyK5PjiWZd+QQIh+1jhdb9LpxTF0qs7Pm8o5QHYZ0M3vKqSqzsZaEB6oWlxZ+q2sJBMI/Ktgd2N5ZwQoRHfg=="], @@ -2593,6 +2590,8 @@ "proxy-from-env": ["proxy-from-env@1.1.0", "", {}, "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg=="], + "pump": ["pump@3.0.3", "", { "dependencies": { "end-of-stream": "^1.1.0", "once": "^1.3.1" } }, "sha512-todwxLMY7/heScKmntwQG8CXVkWUOdYxIvY2s0VWAAMh/nd8SoYiRaKjlr7+iCs984f2P8zvrfWcDDYVb73NfA=="], + "punycode": ["punycode@2.3.1", "", {}, "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg=="], "punycode.js": ["punycode.js@2.3.1", "", {}, "sha512-uxFIHU0YlHYhDQtV4R9J6a52SLx28BCjT+4ieh7IGbgwVJWO+km431c4yRlREUAsAmt/uMjQUyQHNEPf0M39CA=="], @@ -2619,8 +2618,6 @@ "react": ["react@19.2.4", "", {}, "sha512-9nfp2hYpCwOjAN+8TZFGhtWEwgvWHXqESH8qT89AT/lWklpLON22Lc8pEtnpsZz7VmawabSU0gCjnj8aC0euHQ=="], - "react-day-picker": ["react-day-picker@9.13.2", "", { "dependencies": { "@date-fns/tz": "^1.4.1", "date-fns": "^4.1.0", "date-fns-jalali": "^4.1.0-0" }, "peerDependencies": { "react": ">=16.8.0" } }, "sha512-IMPiXfXVIAuR5Yk58DDPBC8QKClrhdXV+Tr/alBrwrHUw0qDDYB1m5zPNuTnnPIr/gmJ4ChMxmtqPdxm8+R4Eg=="], - "react-dom": ["react-dom@19.2.4", "", { "dependencies": { "scheduler": "^0.27.0" }, "peerDependencies": { "react": "^19.2.4" } }, "sha512-AXJdLo8kgMbimY95O2aKQqsz2iWi9jMgKJhRBAxECE4IFxfcazB2LmzloIoibJI3C12IlY20+KFaLv+71bUJeQ=="], "react-is": ["react-is@16.13.1", "", {}, "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ=="], @@ -2711,6 +2708,8 @@ "reusify": ["reusify@1.1.0", "", {}, "sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw=="], + "rfc4648": ["rfc4648@1.5.4", "", {}, "sha512-rRg/6Lb+IGfJqO05HZkN50UtY7K/JhxJag1kP23+zyMfrvoB0B7RWv06MbOzoc79RgCdNTiUaNsTT1AJZ7Z+cg=="], + "rfdc": ["rfdc@1.4.1", "", {}, "sha512-q1b3N5QkRUWUl7iyylaaj3kOpIT0N2i9MqIEQXP73GVsN9cw3fdx8X63cEmWhJGi2PPCF23Ijp7ktmd39rawIA=="], "rimraf": ["rimraf@5.0.10", "", { "dependencies": { "glob": "^10.3.7" }, "bin": { "rimraf": "dist/esm/bin.mjs" } }, "sha512-l0OE8wL34P4nJH/H2ffoaniAokM2qSmrtXHmlpvYr5AVVX8msAyW0l8NVJFDxlSK4u3Uh/f41cQheDVdnYijwQ=="], @@ -2745,8 +2744,6 @@ "schema-utils": ["schema-utils@4.3.3", "", { "dependencies": { "@types/json-schema": "^7.0.9", "ajv": "^8.9.0", "ajv-formats": "^2.1.1", "ajv-keywords": "^5.1.0" } }, "sha512-eflK8wEtyOE6+hsaRVPxvUKYCpRgzLqDTb8krvAsRIwOGlHoSgYLgBXoubGgLd2fT41/OUYdb48v4k4WWHQurA=="], - "secure-json-parse": ["secure-json-parse@2.7.0", "", {}, "sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw=="], - "semver": ["semver@7.7.2", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-RF0Fw+rO5AMf9MAyaRXI4AV0Ulj5lMHqVxxdSgiVbixSCXoEmmX/jk0CuJw4+3SqroYO9VoUh+HcuJivvtJemA=="], "send": ["send@1.2.1", "", { "dependencies": { "debug": "^4.4.3", "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "etag": "^1.8.1", "fresh": "^2.0.0", "http-errors": "^2.0.1", "mime-types": "^3.0.2", "ms": "^2.1.3", "on-finished": "^2.4.1", "range-parser": "^1.2.1", "statuses": "^2.0.2" } }, "sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ=="], @@ -2825,12 +2822,16 @@ "stop-iteration-iterator": ["stop-iteration-iterator@1.1.0", "", { "dependencies": { "es-errors": "^1.3.0", "internal-slot": "^1.1.0" } }, "sha512-eLoXW/DHyl62zxY4SCaIgnRhuMr6ri4juEYARS8E6sCEqzKpOiE521Ucofdx+KnDZl5xmvGYaaKCk5FEOxJCoQ=="], + "stream-buffers": ["stream-buffers@3.0.3", "", {}, "sha512-pqMqwQCso0PBJt2PQmDO0cFj0lyqmiwOMiMSkVtRokl7e+ZTRYgDHKnuZNbqjiJXgsg4nuqtD/zxuo9KqTp0Yw=="], + "stream-chain": ["stream-chain@2.2.5", "", {}, "sha512-1TJmBx6aSWqZ4tx7aTpBDXK0/e2hhcNSTV8+CbFJtDjbb+I1mZ8lHit0Grw9GRT+6JbIrrDd8esncgBi8aBXGA=="], "stream-json": ["stream-json@1.9.1", "", { "dependencies": { "stream-chain": "^2.2.5" } }, "sha512-uWkjJ+2Nt/LO9Z/JyKZbMusL8Dkh97uUBTv3AJQ74y07lVahLY4eEFsPsE97pxYBwr8nnjMAIch5eqI0gPShyw=="], "streamsearch": ["streamsearch@1.1.0", "", {}, "sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg=="], + "streamx": ["streamx@2.23.0", "", { "dependencies": { "events-universal": "^1.0.0", "fast-fifo": "^1.3.2", "text-decoder": "^1.1.0" } }, "sha512-kn+e44esVfn2Fa/O0CPFcex27fjIL6MkVae0Mm6q+E6f0hWv578YCERbv+4m02cjxvDsPKLnmxral/rR6lBMAg=="], + "strict-uri-encode": ["strict-uri-encode@2.0.0", "", {}, "sha512-QwiXZgpRcKkhTj2Scnn++4PKtWsH0kpzZ62L2R6c/LUVYv7hVnZqcg2+sMuT6R7Jusu1vviK/MFsu6kNJfWlEQ=="], "string-argv": ["string-argv@0.3.2", "", {}, "sha512-aqD2Q0144Z+/RqG52NeHEkZauTAUWJO8c6yTftGJKO3Tja5tUgIfmIl6kExvhtxSDP7fXB6DvzkfMpCd/F3G+Q=="], @@ -2899,10 +2900,16 @@ "tapable": ["tapable@2.3.0", "", {}, "sha512-g9ljZiwki/LfxmQADO3dEY1CbpmXT5Hm2fJ+QaGKwSXUylMybePR7/67YW7jOrrvjEgL1Fmz5kzyAjWVWLlucg=="], + "tar-fs": ["tar-fs@3.1.1", "", { "dependencies": { "pump": "^3.0.0", "tar-stream": "^3.1.5" }, "optionalDependencies": { "bare-fs": "^4.0.1", "bare-path": "^3.0.0" } }, "sha512-LZA0oaPOc2fVo82Txf3gw+AkEd38szODlptMYejQUhndHMLQ9M059uXR+AfS7DNo0NpINvSqDsvyaCrBVkptWg=="], + + "tar-stream": ["tar-stream@3.1.7", "", { "dependencies": { "b4a": "^1.6.4", "fast-fifo": "^1.2.0", "streamx": "^2.15.0" } }, "sha512-qJj60CXt7IU1Ffyc3NJMjh6EkuCFej46zUqJ4J7pqYlThyd9bO0XBTmcOIhSzZJVWfsLks0+nle/j538YAW9RQ=="], + "terser": ["terser@5.46.0", "", { "dependencies": { "@jridgewell/source-map": "^0.3.3", "acorn": "^8.15.0", "commander": "^2.20.0", "source-map-support": "~0.5.20" }, "bin": { "terser": "bin/terser" } }, "sha512-jTwoImyr/QbOWFFso3YoU3ik0jBBDJ6JTOQiy/J2YxVJdZCc+5u7skhNwiOR3FQIygFqVUPHl7qbbxtjW2K3Qg=="], "terser-webpack-plugin": ["terser-webpack-plugin@5.3.16", "", { "dependencies": { "@jridgewell/trace-mapping": "^0.3.25", "jest-worker": "^27.4.5", "schema-utils": "^4.3.0", "serialize-javascript": "^6.0.2", "terser": "^5.31.1" }, "peerDependencies": { "webpack": "^5.1.0" } }, "sha512-h9oBFCWrq78NyWWVcSwZarJkZ01c2AyGrzs1crmHZO3QUg9D61Wu4NPjBy69n7JqylFF5y+CsUZYmYEIZ3mR+Q=="], + "text-decoder": ["text-decoder@1.2.3", "", { "dependencies": { "b4a": "^1.6.4" } }, "sha512-3/o9z3X0X0fTupwsYvR03pJ/DjWuqqrfwBgTQzdWDiQSm9KitAyz/9WqsT2JQW7KV2m+bC2ol/zqpW37NHxLaA=="], + "thenify": ["thenify@3.3.1", "", { "dependencies": { "any-promise": "^1.0.0" } }, "sha512-RVZSIV5IG10Hk3enotrhvz0T9em6cyHBLkH/YAZuKqd8hRkKhSfCGIcP2KUY0EPxndzANBmNllzWPwak+bheSw=="], "thenify-all": ["thenify-all@1.6.0", "", { "dependencies": { "thenify": ">= 3.1.0 < 4" } }, "sha512-RNxQH/qI8/t3thXJDwcstUO4zeqo64+Uy/+sNVRBx4Xn2OX+OZ9oP+iJnNFqplFra2ZUVeKCSa2oVWi3T4uVmA=="], @@ -3277,8 +3284,6 @@ "@shipsec/studio-worker/@types/node": ["@types/node@25.2.0", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-DZ8VwRFUNzuqJ5khrvwMXHmvPe+zGayJhr2CDNiKB1WBE1ST8Djl00D0IC4vvNmHMdj6DlbYRIaFE7WHjlDl5w=="], - "@temporalio/common/ms": ["ms@3.0.0-canary.1", "", {}, "sha512-kh8ARjh8rMN7Du2igDRO9QJnqCb2xYTJxyQYK7vJJS4TvLLmsbyhiKpSW+t+y26gyOyMd0riphX0GeWKU3ky5g=="], - "@temporalio/worker/supports-color": ["supports-color@8.1.1", "", { "dependencies": { "has-flag": "^4.0.0" } }, "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q=="], "@testing-library/dom/aria-query": ["aria-query@5.3.0", "", { "dependencies": { "dequal": "^2.0.3" } }, "sha512-b0P0sZPKtyu8HkeRAfCq0IfURZK+SuwMjY1UXGBU27wpAiTwQAIlq56IbIO+ytk/JjS1fMR14ee5WBBfKi5J6A=="], @@ -3297,6 +3302,8 @@ "@types/express-serve-static-core/@types/node": ["@types/node@25.2.0", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-DZ8VwRFUNzuqJ5khrvwMXHmvPe+zGayJhr2CDNiKB1WBE1ST8Djl00D0IC4vvNmHMdj6DlbYRIaFE7WHjlDl5w=="], + "@types/node-fetch/@types/node": ["@types/node@25.2.0", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-DZ8VwRFUNzuqJ5khrvwMXHmvPe+zGayJhr2CDNiKB1WBE1ST8Djl00D0IC4vvNmHMdj6DlbYRIaFE7WHjlDl5w=="], + "@types/node-forge/@types/node": ["@types/node@25.2.0", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-DZ8VwRFUNzuqJ5khrvwMXHmvPe+zGayJhr2CDNiKB1WBE1ST8Djl00D0IC4vvNmHMdj6DlbYRIaFE7WHjlDl5w=="], "@types/pg/@types/node": ["@types/node@25.2.0", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-DZ8VwRFUNzuqJ5khrvwMXHmvPe+zGayJhr2CDNiKB1WBE1ST8Djl00D0IC4vvNmHMdj6DlbYRIaFE7WHjlDl5w=="], @@ -3307,6 +3314,8 @@ "@types/serve-static/@types/node": ["@types/node@25.2.0", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-DZ8VwRFUNzuqJ5khrvwMXHmvPe+zGayJhr2CDNiKB1WBE1ST8Djl00D0IC4vvNmHMdj6DlbYRIaFE7WHjlDl5w=="], + "@types/stream-buffers/@types/node": ["@types/node@25.2.0", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-DZ8VwRFUNzuqJ5khrvwMXHmvPe+zGayJhr2CDNiKB1WBE1ST8Djl00D0IC4vvNmHMdj6DlbYRIaFE7WHjlDl5w=="], + "@types/superagent/@types/node": ["@types/node@25.2.0", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-DZ8VwRFUNzuqJ5khrvwMXHmvPe+zGayJhr2CDNiKB1WBE1ST8Djl00D0IC4vvNmHMdj6DlbYRIaFE7WHjlDl5w=="], "@types/ws/@types/node": ["@types/node@25.2.0", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-DZ8VwRFUNzuqJ5khrvwMXHmvPe+zGayJhr2CDNiKB1WBE1ST8Djl00D0IC4vvNmHMdj6DlbYRIaFE7WHjlDl5w=="], @@ -3349,6 +3358,8 @@ "data-urls/whatwg-mimetype": ["whatwg-mimetype@5.0.0", "", {}, "sha512-sXcNcHOC51uPGF0P/D4NVtrkjSU2fNsm9iog4ZvZJsL3rjoDAzXZhkm2MWt1y+PUdggKAYVoMAIYcs78wJ51Cw=="], + "debug/ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], + "decamelize-keys/decamelize": ["decamelize@1.2.0", "", {}, "sha512-z2S+W9X73hAUUki+N+9Za2lBlun89zigOyGrsax+KUQ6wKW4ZoWpEYBkGhQjwAjjDCkWxhY0VKEhk8wzY7F5cA=="], "decamelize-keys/map-obj": ["map-obj@1.0.1", "", {}, "sha512-7N/q3lyZ+LVCp7PzuxrJr4KMbBE2hW7BT7YNia330OFxIf4d3r5zVpicP2650l7CPN6RM9zOJRl3NGpqSiw3Eg=="], @@ -3367,8 +3378,6 @@ "express/body-parser": ["body-parser@2.2.2", "", { "dependencies": { "bytes": "^3.1.2", "content-type": "^1.0.5", "debug": "^4.4.3", "http-errors": "^2.0.0", "iconv-lite": "^0.7.0", "on-finished": "^2.4.1", "qs": "^6.14.1", "raw-body": "^3.0.1", "type-is": "^2.0.1" } }, "sha512-oP5VkATKlNwcgvxi0vM0p/D3n2C3EReYVX+DNYs5TjZFn/oQt2j+4sVJtSMr18pdRr8wjTcBl6LoV+FUwzPmNA=="], - "express/cookie-signature": ["cookie-signature@1.2.2", "", {}, "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg=="], - "fast-glob/glob-parent": ["glob-parent@5.1.2", "", { "dependencies": { "is-glob": "^4.0.1" } }, "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow=="], "foreground-child/signal-exit": ["signal-exit@4.1.0", "", {}, "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw=="], @@ -3485,6 +3494,8 @@ "schema-utils/ajv-formats": ["ajv-formats@2.1.1", "", { "dependencies": { "ajv": "^8.0.0" } }, "sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA=="], + "send/ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], + "slice-ansi/ansi-styles": ["ansi-styles@6.2.3", "", {}, "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg=="], "source-map-loader/iconv-lite": ["iconv-lite@0.6.3", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw=="], @@ -3501,8 +3512,6 @@ "sucrase/commander": ["commander@4.1.1", "", {}, "sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA=="], - "supertest/cookie-signature": ["cookie-signature@1.2.2", "", {}, "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg=="], - "tailwindcss/postcss-selector-parser": ["postcss-selector-parser@6.1.2", "", { "dependencies": { "cssesc": "^3.0.0", "util-deprecate": "^1.0.2" } }, "sha512-Q8qQfPiZ+THO/3ZrOrO0cJJKfpYCagtMUkXbnEfmgUjwXg6z/WBeOyS9APBBPCTSiDV+s4SwQGu8yFsiMRIudg=="], "tailwindcss/resolve": ["resolve@1.22.11", "", { "dependencies": { "is-core-module": "^2.16.1", "path-parse": "^1.0.7", "supports-preserve-symlinks-flag": "^1.0.0" }, "bin": { "resolve": "bin/resolve" } }, "sha512-RfqAvLnMl313r7c9oclB1HhUEAezcpLjz95wFH4LVuhk9JF/r22qmVP9AMmOU4vMX7Q8pN8jwNg/CSpdFnMjTQ=="], @@ -3633,14 +3642,20 @@ "@pm2/agent/chalk/supports-color": ["supports-color@7.2.0", "", { "dependencies": { "has-flag": "^4.0.0" } }, "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw=="], + "@pm2/agent/debug/ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], + "@pm2/agent/semver/lru-cache": ["lru-cache@6.0.0", "", { "dependencies": { "yallist": "^4.0.0" } }, "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA=="], "@pm2/io/async/lodash": ["lodash@4.17.23", "", {}, "sha512-LgVTMpQtIopCi79SJeDiP0TfWi5CNEc/L/aRdTh3yIvmZXTnheWpKjSZhnvMl8iXbC1tFg9gdHHDMLoV7CnG+w=="], + "@pm2/io/debug/ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], + "@pm2/io/semver/lru-cache": ["lru-cache@6.0.0", "", { "dependencies": { "yallist": "^4.0.0" } }, "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA=="], "@pm2/js-api/async/lodash": ["lodash@4.17.23", "", {}, "sha512-LgVTMpQtIopCi79SJeDiP0TfWi5CNEc/L/aRdTh3yIvmZXTnheWpKjSZhnvMl8iXbC1tFg9gdHHDMLoV7CnG+w=="], + "@pm2/js-api/debug/ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], + "@redocly/openapi-core/minimatch/brace-expansion": ["brace-expansion@2.0.2", "", { "dependencies": { "balanced-match": "^1.0.0" } }, "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ=="], "@shipsec/component-sdk/@types/node/undici-types": ["undici-types@6.21.0", "", {}, "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ=="], @@ -3763,6 +3778,8 @@ "multer/type-is/mime-types": ["mime-types@2.1.35", "", { "dependencies": { "mime-db": "1.52.0" } }, "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw=="], + "needle/debug/ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], + "node-fetch/whatwg-url/tr46": ["tr46@0.0.3", "", {}, "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw=="], "node-fetch/whatwg-url/webidl-conversions": ["webidl-conversions@3.0.1", "", {}, "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ=="], @@ -3801,6 +3818,8 @@ "@nestjs/platform-express/express/send/mime": ["mime@1.6.0", "", { "bin": { "mime": "cli.js" } }, "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg=="], + "@nestjs/platform-express/express/send/ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], + "@nestjs/platform-express/express/type-is/media-typer": ["media-typer@0.3.0", "", {}, "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ=="], "@nestjs/platform-express/express/type-is/mime-types": ["mime-types@2.1.35", "", { "dependencies": { "mime-db": "1.52.0" } }, "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw=="], diff --git a/packages/component-sdk/src/runner.ts b/packages/component-sdk/src/runner.ts index dd366bbdb..828af2b56 100644 --- a/packages/component-sdk/src/runner.ts +++ b/packages/component-sdk/src/runner.ts @@ -516,6 +516,29 @@ async function runDockerWithPty( }); } +/** + * Override hook for the docker runner. When set, all `kind: 'docker'` executions + * are routed through this function instead of the built-in runComponentInDocker. + * + * Used by the worker to plug in a K8s Job runner at startup: + * setDockerRunnerOverride(runComponentInK8sJob) + */ +type DockerRunnerOverrideFn = ( + runner: DockerRunnerConfig, + params: I, + context: ExecutionContext, +) => Promise; + +let dockerRunnerOverride: DockerRunnerOverrideFn | null = null; + +export function setDockerRunnerOverride(fn: DockerRunnerOverrideFn): void { + dockerRunnerOverride = fn; +} + +export function clearDockerRunnerOverride(): void { + dockerRunnerOverride = null; +} + export async function runComponentWithRunner( runner: RunnerConfig, execute: (params: I, context: ExecutionContext) => Promise, @@ -526,6 +549,9 @@ export async function runComponentWithRunner( case 'inline': return runComponentInline(execute, params, context); case 'docker': + if (dockerRunnerOverride) { + return dockerRunnerOverride(runner, params, context); + } return runComponentInDocker(runner, params, context); case 'remote': context.logger.info(`[Runner] remote execution stub for ${runner.endpoint}`); diff --git a/worker/package.json b/worker/package.json index bc75fb3da..055576be9 100644 --- a/worker/package.json +++ b/worker/package.json @@ -25,9 +25,9 @@ "@aws-sdk/client-s3": "^3.975.0", "@googleapis/admin": "^29.0.0", "@grpc/grpc-js": "^1.14.3", + "@kubernetes/client-node": "^1.4.0", "@modelcontextprotocol/sdk": "^1.25.1", "@okta/okta-sdk-nodejs": "^7.3.0", - "@opensearch-project/opensearch": "^3.5.1", "@shipsec/component-sdk": "*", "@shipsec/contracts": "*", "@shipsec/shared": "*", diff --git a/worker/src/temporal/workers/dev.worker.ts b/worker/src/temporal/workers/dev.worker.ts index 6e2495cc6..e8ed03b8c 100644 --- a/worker/src/temporal/workers/dev.worker.ts +++ b/worker/src/temporal/workers/dev.worker.ts @@ -29,7 +29,7 @@ import { registerComponentToolActivity, registerLocalMcpActivity, registerRemoteMcpActivity, - cleanupRunResourcesActivity, + cleanupLocalMcpActivity, prepareAndRegisterToolActivity, areAllToolsReadyActivity, } from '../activities/mcp.activity'; @@ -56,7 +56,6 @@ import { ConfigurationError } from '@shipsec/component-sdk'; import { getTopicResolver } from '../../common/kafka-topic-resolver'; import * as schema from '../../adapters/schema'; import { logHeartbeat } from '../../utils/debug-logger'; -import { validateWorkerEnv } from '../../config/env.validate'; // Load environment variables from instance-specific env if set, otherwise fall back // to the worker's default `.env`. @@ -67,7 +66,6 @@ const instanceEnvPath = instanceNum : undefined; config({ path: instanceEnvPath ?? join(workerRoot, '.env') }); -validateWorkerEnv(process.env); if (typeof globalThis.crypto === 'undefined') { Object.defineProperty(globalThis, 'crypto', { @@ -232,6 +230,14 @@ async function main() { console.log(`✅ Service adapters initialized`); + // Register K8s runner override if EXECUTION_MODE=k8s + if (process.env.EXECUTION_MODE === 'k8s') { + const { runComponentInK8sJob } = await import('../../utils/k8s-runner'); + const { setDockerRunnerOverride } = await import('@shipsec/component-sdk'); + setDockerRunnerOverride(runComponentInK8sJob); + console.log('[Worker] K8s execution mode enabled — docker runner overridden with K8s Jobs'); + } + console.log(`🏗️ Creating Temporal worker...`); console.log( ` - Activities: ${Object.keys({ @@ -245,7 +251,7 @@ async function main() { registerComponentToolActivity, registerLocalMcpActivity, registerRemoteMcpActivity, - cleanupRunResourcesActivity, + cleanupLocalMcpActivity, discoverMcpToolsActivity, discoverMcpGroupToolsActivity, cacheDiscoveryResultActivity, @@ -285,7 +291,7 @@ async function main() { registerComponentToolActivity, registerLocalMcpActivity, registerRemoteMcpActivity, - cleanupRunResourcesActivity, + cleanupLocalMcpActivity, prepareAndRegisterToolActivity, areAllToolsReadyActivity, discoverMcpToolsActivity, diff --git a/worker/src/utils/index.ts b/worker/src/utils/index.ts index 4afa27813..909c6e942 100644 --- a/worker/src/utils/index.ts +++ b/worker/src/utils/index.ts @@ -2,4 +2,9 @@ * Utility exports for worker components */ -export { IsolatedContainerVolume, cleanupOrphanedVolumes } from './isolated-volume'; +export { + IsolatedContainerVolume, + cleanupOrphanedVolumes, + createIsolatedVolume, +} from './isolated-volume'; +export { IsolatedK8sVolume } from './k8s-volume'; diff --git a/worker/src/utils/isolated-volume.ts b/worker/src/utils/isolated-volume.ts index ea6304709..50400b571 100644 --- a/worker/src/utils/isolated-volume.ts +++ b/worker/src/utils/isolated-volume.ts @@ -2,6 +2,7 @@ import { spawn } from 'child_process'; import { promisify } from 'util'; import { exec as execCallback } from 'child_process'; import { ValidationError, ConfigurationError, ContainerError } from '@shipsec/component-sdk'; +import { IsolatedK8sVolume } from './k8s-volume'; const exec = promisify(execCallback); @@ -505,6 +506,9 @@ export class IsolatedContainerVolume { * ``` */ export async function cleanupOrphanedVolumes(olderThanHours = 24): Promise { + // In K8s mode volumes are ConfigMaps — no Docker daemon to query + if (process.env.EXECUTION_MODE === 'k8s') return 0; + try { const { stdout } = await exec( 'docker volume ls --filter "label=studio.managed=true" --format "{{.Name}}|||{{.CreatedAt}}"', @@ -542,3 +546,20 @@ export async function cleanupOrphanedVolumes(olderThanHours = 24): Promise; // mountPath → configMapName +} + +// Lazy-init shared K8s clients +let _kc: k8s.KubeConfig | null = null; +let _batchApi: k8s.BatchV1Api | null = null; +let _coreApi: k8s.CoreV1Api | null = null; + +function getKubeConfig(): k8s.KubeConfig { + if (!_kc) { + _kc = new k8s.KubeConfig(); + _kc.loadFromCluster(); // uses in-cluster SA token + } + return _kc; +} + +function getBatchApi(): k8s.BatchV1Api { + if (!_batchApi) _batchApi = getKubeConfig().makeApiClient(k8s.BatchV1Api); + return _batchApi; +} + +function getCoreApi(): k8s.CoreV1Api { + if (!_coreApi) _coreApi = getKubeConfig().makeApiClient(k8s.CoreV1Api); + return _coreApi; +} + +function getJobNamespace(): string { + return process.env.K8S_JOB_NAMESPACE || 'shipsec-workloads'; +} + +function sanitizeName(raw: string): string { + // K8s names: lowercase, alphanumeric + hyphens, max 63 chars + return raw + .toLowerCase() + .replace(/[^a-z0-9-]/g, '-') + .replace(/-+/g, '-') + .replace(/^-|-$/g, '') + .slice(0, 53); // leave room for random suffix +} + +function generateJobName(context: ExecutionContext, image: string): string { + const imgShort = sanitizeName(image.split('/').pop()?.split(':')[0] || 'job'); + const runShort = sanitizeName(context.runId).slice(0, 8); + const rand = Math.random().toString(36).slice(2, 8); + return `ss-${imgShort}-${runShort}-${rand}`; +} + +/** + * Shell snippet that captures files from writable volume mounts. + * Reads $SHIPSEC_WRITABLE_MOUNTS (space-separated paths) and emits + * each file as base64 between markers so the worker can parse them + * from pod logs and write them back to their backing ConfigMaps. + */ +const VOLUME_CAPTURE_SCRIPT = [ + `echo '${VOLUME_DELIMITER}'`, + 'for __mp in $SHIPSEC_WRITABLE_MOUNTS; do', + ' find "$__mp" -type f 2>/dev/null | while IFS= read -r __f; do', + ' __rel="${__f#$__mp/}"', + ' echo "___FILE_START___:$__mp:$__rel"', + ' base64 "$__f" 2>/dev/null || true', + ' echo "___FILE_END___"', + ' done', + 'done', +].join('; '); + +/** + * Build the command wrapper that emits the output file to stdout. + * + * For images with a shell: wraps original command so that after it exits, + * the output file is printed to stdout with a delimiter prefix. + * If writable volumes exist, also captures their contents as base64. + * + * For images without a shell (distroless): returns original command as-is, + * relying on stdout-based output fallback. + */ +function wrapCommandForOutput(runner: DockerRunnerConfig): { command: string[]; args: string[] } { + const { entrypoint, command } = runner; + + // Volume capture suffix — only emits data when SHIPSEC_WRITABLE_MOUNTS is set + const volCapture = `; if [ -n "$SHIPSEC_WRITABLE_MOUNTS" ]; then ${VOLUME_CAPTURE_SCRIPT}; fi`; + + const isShellEntrypoint = + entrypoint === 'sh' || + entrypoint === 'bash' || + entrypoint === '/bin/sh' || + entrypoint === '/bin/bash'; + + if (isShellEntrypoint && command.length >= 2 && command[0] === '-c') { + // Shell wrapper pattern: entrypoint=sh, command=['-c', 'binary "$@"', '--', ...dynamicArgs] + const shellScript = command[1]; + const dynamicArgsMatch = shellScript.match(/^(\S+)\s+"\$@"$/); + + if (dynamicArgsMatch) { + // Dynamic args pattern for distroless images (e.g., 'subfinder "$@"') + // These images don't have sh — use their default ENTRYPOINT directly. + // The dynamic args follow after '--' in the command array. + const dashDashIdx = command.indexOf('--'); + const dynamicArgs = dashDashIdx >= 0 ? command.slice(dashDashIdx + 1) : []; + // Return empty command to use image's ENTRYPOINT, pass dynamic args directly + return { command: [], args: dynamicArgs }; + } + + // Regular shell script — wrap with output capture + const userScript = command.slice(1).join(' '); + const wrapped = `${userScript}; __exit=$?; echo '${OUTPUT_DELIMITER}'; cat ${CONTAINER_OUTPUT_PATH}/${OUTPUT_FILENAME} 2>/dev/null || echo '{}'${volCapture}; exit $__exit`; + return { command: [entrypoint!], args: ['-c', wrapped] }; + } + + if (isShellEntrypoint) { + return { command: [entrypoint!], args: command }; + } + + // For non-shell entrypoints (e.g., 'httpx', 'nuclei', binary entrypoints): + // Use the entrypoint directly — the image may be distroless (no /bin/sh). + // Output is captured from stdout via parseOutputFromLogs fallback. + if (entrypoint) { + return { command: [entrypoint], args: command }; + } + + if (command.length > 0) { + return { command: [command[0]], args: command.slice(1) }; + } + + return { command: [], args: [] }; +} + +/** + * Create a ConfigMap containing the serialized input data. + */ +async function createInputConfigMap( + name: string, + namespace: string, + inputData: unknown, +): Promise { + const core = getCoreApi(); + const body: k8s.V1ConfigMap = { + metadata: { + name, + namespace, + labels: { + 'app.kubernetes.io/managed-by': 'shipsec-worker', + 'shipsec.ai/purpose': 'job-input', + }, + }, + data: { + 'input.json': JSON.stringify(inputData), + }, + }; + await core.createNamespacedConfigMap({ namespace, body }); +} + +/** + * Build the K8s Job spec from a DockerRunnerConfig. + */ +function buildJobSpec( + jobName: string, + namespace: string, + configMapName: string, + runner: DockerRunnerConfig, + context: ExecutionContext, +): BuildJobResult { + const { command, args } = wrapCommandForOutput(runner); + const timeoutSeconds = runner.timeoutSeconds || 300; + + // Track writable ConfigMap volumes for post-execution data capture + const writableVolumeMappings = new Map(); + + // Build env vars + const envVars: k8s.V1EnvVar[] = [ + { name: 'SHIPSEC_INPUT_PATH', value: '/shipsec-input/input.json' }, + { name: 'SHIPSEC_OUTPUT_PATH', value: `${CONTAINER_OUTPUT_PATH}/${OUTPUT_FILENAME}` }, + ]; + if (runner.env) { + for (const [key, value] of Object.entries(runner.env)) { + // Override HOME=/root for distroless images — /root is not writable + if (key === 'HOME' && value === '/root') { + envVars.push({ name: key, value: '/tmp' }); + } else { + envVars.push({ name: key, value }); + } + } + } + + // Build volume mounts + const volumeMounts: k8s.V1VolumeMount[] = [ + { name: 'input', mountPath: '/shipsec-input', readOnly: true }, + { name: 'output', mountPath: CONTAINER_OUTPUT_PATH }, + ]; + + const volumes: k8s.V1Volume[] = [ + { + name: 'input', + configMap: { name: configMapName }, + }, + { + name: 'output', + emptyDir: {}, + }, + ]; + + // Handle additional volumes (from IsolatedK8sVolume) + if (runner.volumes) { + for (let i = 0; i < runner.volumes.length; i++) { + const vol = runner.volumes[i]; + if (!vol || !vol.source || !vol.target) continue; + + const volName = `extra-vol-${i}`; + + if (vol.source.startsWith('configmap:') && (vol.readOnly ?? true)) { + // ConfigMap-backed volume from IsolatedK8sVolume (read-only) + const cmName = vol.source.replace('configmap:', ''); + volumes.push({ + name: volName, + configMap: { name: cmName }, + }); + } else if (vol.source.startsWith('configmap:') && !(vol.readOnly ?? true)) { + const cmName = vol.source.replace('configmap:', ''); + // Use emptyDir for the actual mount (ConfigMaps are read-only in K8s) + volumes.push({ + name: volName, + emptyDir: {}, + }); + // Track for post-execution data capture + writableVolumeMappings.set(vol.target, cmName); + } else { + // Treat as emptyDir (can't use host paths in K8s Jobs) + volumes.push({ + name: volName, + emptyDir: {}, + }); + } + + volumeMounts.push({ + name: volName, + mountPath: vol.target, + readOnly: vol.readOnly ?? false, + }); + } + } + + // Add env var for writable mount paths so the shell wrapper can capture files + if (writableVolumeMappings.size > 0) { + envVars.push({ + name: 'SHIPSEC_WRITABLE_MOUNTS', + value: Array.from(writableVolumeMappings.keys()).join(' '), + }); + } + + const job: k8s.V1Job = { + metadata: { + name: jobName, + namespace, + labels: { + 'app.kubernetes.io/managed-by': 'shipsec-worker', + 'shipsec.ai/run-id': sanitizeName(context.runId), + 'shipsec.ai/component-ref': sanitizeName(context.componentRef), + }, + }, + spec: { + backoffLimit: 0, // no retries — Temporal handles retry logic + activeDeadlineSeconds: timeoutSeconds, + ttlSecondsAfterFinished: 120, // auto-cleanup after 2 min + template: { + metadata: { + labels: { + 'app.kubernetes.io/managed-by': 'shipsec-worker', + 'shipsec.ai/run-id': sanitizeName(context.runId), + }, + }, + spec: { + restartPolicy: 'Never', + ...(process.env.K8S_IMAGE_PULL_SECRET + ? { imagePullSecrets: [{ name: process.env.K8S_IMAGE_PULL_SECRET }] } + : {}), + containers: [ + { + name: 'component', + image: runner.image, + imagePullPolicy: + (process.env.K8S_JOB_IMAGE_PULL_POLICY as 'Always' | 'IfNotPresent' | 'Never') || + 'IfNotPresent', + command: command.length > 0 ? command : undefined, + args: args.length > 0 ? args : undefined, + env: envVars, + volumeMounts, + resources: { + requests: { cpu: '100m', memory: '128Mi' }, + limits: { cpu: '1000m', memory: '2Gi' }, + }, + }, + ], + volumes, + }, + }, + }, + }; + + return { job, writableVolumeMappings }; +} + +/** + * Wait for a Job to complete (or fail/timeout). + * Returns the pod name for log retrieval. + */ +async function waitForJobCompletion( + jobName: string, + namespace: string, + timeoutMs: number, + context: ExecutionContext, +): Promise<{ podName: string; succeeded: boolean }> { + const batch = getBatchApi(); + const core = getCoreApi(); + const deadline = Date.now() + timeoutMs; + + // Find the pod created by this Job + let podName = ''; + while (!podName && Date.now() < deadline) { + const pods = await core.listNamespacedPod({ + namespace, + labelSelector: `job-name=${jobName}`, + }); + if (pods.items.length > 0) { + podName = pods.items[0].metadata?.name || ''; + } + if (!podName) { + await new Promise((r) => setTimeout(r, 1000)); + } + } + + if (!podName) { + throw new TimeoutError(`Timed out waiting for Job pod to appear: ${jobName}`, timeoutMs); + } + + context.logger.info(`[K8sRunner] Job ${jobName} → pod ${podName}`); + + // Stream logs in real-time while waiting + const logPromise = streamPodLogs(podName, namespace, context).catch((err) => { + context.logger.warn(`[K8sRunner] Log streaming error: ${err.message}`); + }); + + // Poll Job status until done + while (Date.now() < deadline) { + const job = await batch.readNamespacedJob({ name: jobName, namespace }); + const status = job.status; + + if (status?.succeeded && status.succeeded > 0) { + await logPromise; + return { podName, succeeded: true }; + } + if (status?.failed && status.failed > 0) { + await logPromise; + return { podName, succeeded: false }; + } + + await new Promise((r) => setTimeout(r, 2000)); + } + + throw new TimeoutError(`Job ${jobName} timed out after ${timeoutMs / 1000}s`, timeoutMs, { + details: { jobName, podName }, + }); +} + +/** + * Stream pod logs to the context logger and terminal collector. + * Uses the K8s Log API with a writable stream to capture output in real-time. + */ +async function streamPodLogs( + podName: string, + namespace: string, + context: ExecutionContext, +): Promise { + const kc = getKubeConfig(); + const log = new k8s.Log(kc); + + const { PassThrough } = await import('stream'); + const logStream = new PassThrough(); + + logStream.on('data', (chunk: Buffer) => { + const text = chunk.toString(); + // Feed to terminal collector for real-time UI streaming + if (context.terminalCollector) { + context.terminalCollector({ + runId: context.runId, + nodeRef: context.componentRef, + stream: 'stdout', + chunkIndex: 0, + payload: text, + recordedAt: new Date().toISOString(), + deltaMs: 0, + origin: 'k8s-job', + }); + } + // Also feed to log collector + if (context.logCollector) { + context.logCollector({ + runId: context.runId, + nodeRef: context.componentRef, + stream: 'stdout', + level: 'info', + message: text, + timestamp: new Date().toISOString(), + }); + } + }); + + try { + await log.log(namespace, podName, 'component', logStream, { + follow: true, + pretty: false, + timestamps: false, + }); + } catch (err) { + context.logger.warn(`[K8sRunner] Log streaming failed: ${(err as Error).message}`); + } +} + +/** + * Read final pod logs after completion. + */ +async function readPodLogs(podName: string, namespace: string): Promise { + const core = getCoreApi(); + const logResponse = await core.readNamespacedPodLog({ + name: podName, + namespace, + container: 'component', + }); + // The response can be a string directly + return typeof logResponse === 'string' ? logResponse : String(logResponse); +} + +/** + * Parse the volume data section from pod logs. + * Returns a nested map: mountPath -> (relativeFilePath -> base64Content). + */ +function extractVolumeDataFromLogs(logs: string): Map> { + const FILE_START = '___FILE_START___:'; + const FILE_END = '___FILE_END___'; + + const result = new Map>(); + + const volIdx = logs.lastIndexOf(VOLUME_DELIMITER); + if (volIdx === -1) return result; + + const volSection = logs.slice(volIdx + VOLUME_DELIMITER.length); + const lines = volSection.split('\n'); + + let currentMount = ''; + let currentFile = ''; + let currentData: string[] = []; + let inFile = false; + + for (const line of lines) { + if (line.startsWith(FILE_START)) { + // Parse mount path and relative path + const rest = line.slice(FILE_START.length); + const firstColon = rest.indexOf(':'); + if (firstColon === -1) continue; + currentMount = rest.slice(0, firstColon); + currentFile = rest.slice(firstColon + 1); + currentData = []; + inFile = true; + } else if (line.trim() === FILE_END && inFile) { + // Save the file + if (!result.has(currentMount)) { + result.set(currentMount, new Map()); + } + result.get(currentMount)!.set(currentFile, currentData.join('\n')); + inFile = false; + } else if (inFile) { + currentData.push(line); + } + } + + return result; +} + +/** + * Write captured volume data back to their backing ConfigMaps. + * This allows volume.readFiles() to access output data after the pod terminates. + */ +async function writeBackVolumeData( + volumeData: Map>, + writableVolumeMappings: Map, + namespace: string, + context: ExecutionContext, +): Promise { + const core = getCoreApi(); + + for (const [mountPath, files] of volumeData) { + const cmName = writableVolumeMappings.get(mountPath); + if (!cmName) continue; + + const binaryData: Record = {}; + + for (const [relPath, base64Content] of files) { + // Flatten path separators same as IsolatedK8sVolume + const key = relPath.replace(/\//g, '__'); + // Store as binaryData (base64) to handle any file type + binaryData[key] = base64Content; + } + + try { + // Read existing ConfigMap and merge + const existing = await core.readNamespacedConfigMap({ name: cmName, namespace }); + const body: k8s.V1ConfigMap = { + ...existing, + data: { ...(existing.data || {}) }, + binaryData: { ...(existing.binaryData || {}), ...binaryData }, + }; + await core.replaceNamespacedConfigMap({ name: cmName, namespace, body }); + context.logger.info(`[K8sRunner] Wrote back ${files.size} files to ConfigMap ${cmName}`); + } catch (err) { + context.logger.warn( + `[K8sRunner] Failed to write back volume data to ${cmName}: ${(err as Error).message}`, + ); + } + } +} + +/** + * Parse component output from pod logs. + * Looks for the OUTPUT_DELIMITER marker — everything after it is the JSON output. + * Falls back to parsing the full stdout as JSON. + */ +function parseOutputFromLogs(logs: string, context: ExecutionContext): O { + // Strip volume data section if present (comes after output) + let cleanLogs = logs; + const volIdx = cleanLogs.lastIndexOf(VOLUME_DELIMITER); + if (volIdx !== -1) { + cleanLogs = cleanLogs.slice(0, volIdx); + } + + // Look for the output delimiter + const delimiterIdx = cleanLogs.lastIndexOf(OUTPUT_DELIMITER); + if (delimiterIdx !== -1) { + const outputStr = cleanLogs.slice(delimiterIdx + OUTPUT_DELIMITER.length).trim(); + if (outputStr) { + try { + return JSON.parse(outputStr) as O; + } catch (e) { + context.logger.warn( + `[K8sRunner] Failed to parse delimited output: ${(e as Error).message}`, + ); + } + } + } + + // Fallback: try parsing the last line as JSON + const lines = cleanLogs.trim().split('\n'); + for (let i = lines.length - 1; i >= 0; i--) { + const line = lines[i].trim(); + if (line.startsWith('{') || line.startsWith('[')) { + try { + return JSON.parse(line) as O; + } catch { + continue; + } + } + } + + // Fallback: return raw logs as string output + context.logger.warn('[K8sRunner] No structured output found, returning raw stdout'); + return cleanLogs.trim() as unknown as O; +} + +/** + * Clean up resources created for a Job execution. + */ +async function cleanup( + jobName: string, + configMapName: string, + namespace: string, + context: ExecutionContext, +): Promise { + const batch = getBatchApi(); + const core = getCoreApi(); + + try { + await batch.deleteNamespacedJob({ + name: jobName, + namespace, + body: { propagationPolicy: 'Background' }, + }); + } catch (err) { + context.logger.warn(`[K8sRunner] Failed to delete Job ${jobName}: ${(err as Error).message}`); + } + + try { + await core.deleteNamespacedConfigMap({ name: configMapName, namespace }); + } catch (err) { + context.logger.warn( + `[K8sRunner] Failed to delete ConfigMap ${configMapName}: ${(err as Error).message}`, + ); + } +} + +/** + * Execute a component as a Kubernetes Job. + * + * Drop-in replacement for runComponentInDocker — same signature, + * registered via setDockerRunnerOverride() at worker startup. + */ +export async function runComponentInK8sJob( + runner: DockerRunnerConfig, + params: I, + context: ExecutionContext, +): Promise { + const namespace = getJobNamespace(); + const jobName = generateJobName(context, runner.image); + const configMapName = `${jobName}-input`; + const timeoutMs = (runner.timeoutSeconds || 300) * 1000; + + context.logger.info( + `[K8sRunner] Creating Job ${jobName} in ${namespace} (image: ${runner.image})`, + ); + context.emitProgress(`Launching K8s Job: ${runner.image}`); + + try { + // 1. Create input ConfigMap + await createInputConfigMap(configMapName, namespace, params); + context.logger.info(`[K8sRunner] Created ConfigMap ${configMapName}`); + + // 2. Create Job + const { job: jobSpec, writableVolumeMappings } = buildJobSpec( + jobName, + namespace, + configMapName, + runner, + context, + ); + await getBatchApi().createNamespacedJob({ namespace, body: jobSpec }); + context.logger.info(`[K8sRunner] Created Job ${jobName}`); + + // 3. Wait for completion + const { podName, succeeded } = await waitForJobCompletion( + jobName, + namespace, + timeoutMs, + context, + ); + + // 4. Read final logs + const logs = await readPodLogs(podName, namespace); + + if (!succeeded) { + context.logger.error(`[K8sRunner] Job ${jobName} failed`); + throw new ContainerError(`K8s Job failed: ${jobName}`, { + details: { jobName, podName, logs: logs.slice(-500) }, + }); + } + + context.logger.info(`[K8sRunner] Job ${jobName} completed successfully`); + context.emitProgress('K8s Job completed'); + + // 4.5. Write back writable volume data to ConfigMaps + // Must happen BEFORE cleanup so volume.readFiles() can access updated ConfigMaps + if (writableVolumeMappings.size > 0) { + const volumeData = extractVolumeDataFromLogs(logs); + if (volumeData.size > 0) { + await writeBackVolumeData(volumeData, writableVolumeMappings, namespace, context); + } + } + + // 5. Parse output + return parseOutputFromLogs(logs, context); + } finally { + // 6. Cleanup + await cleanup(jobName, configMapName, namespace, context); + } +} diff --git a/worker/src/utils/k8s-volume.ts b/worker/src/utils/k8s-volume.ts new file mode 100644 index 000000000..248a7e3df --- /dev/null +++ b/worker/src/utils/k8s-volume.ts @@ -0,0 +1,235 @@ +/** + * IsolatedK8sVolume — K8s-native replacement for IsolatedContainerVolume. + * + * Uses ConfigMaps instead of Docker named volumes. Same interface as + * IsolatedContainerVolume so components can swap transparently. + * + * Limits: + * - ConfigMap total size: 1 MiB (sufficient for target lists, configs, templates) + * - For binary data or large payloads, consider using a PVC-based approach + */ +import * as k8s from '@kubernetes/client-node'; +import { ValidationError, ConfigurationError, ContainerError } from '@shipsec/component-sdk'; + +let _kc: k8s.KubeConfig | null = null; +let _coreApi: k8s.CoreV1Api | null = null; + +function getKubeConfig(): k8s.KubeConfig { + if (!_kc) { + _kc = new k8s.KubeConfig(); + _kc.loadFromCluster(); + } + return _kc; +} + +function getCoreApi(): k8s.CoreV1Api { + if (!_coreApi) _coreApi = getKubeConfig().makeApiClient(k8s.CoreV1Api); + return _coreApi; +} + +function getNamespace(): string { + return process.env.K8S_JOB_NAMESPACE || 'shipsec-workloads'; +} + +function sanitizeName(raw: string): string { + return raw + .toLowerCase() + .replace(/[^a-z0-9-]/g, '-') + .replace(/-+/g, '-') + .replace(/^-|-$/g, '') + .slice(0, 53); +} + +export class IsolatedK8sVolume { + private configMapName?: string; + private isInitialized = false; + private namespace: string; + + constructor( + private tenantId: string, + private runId: string, + ) { + if (!/^[a-zA-Z0-9_-]+$/.test(tenantId)) { + throw new ValidationError( + 'Invalid tenant ID: must contain only alphanumeric characters, hyphens, and underscores', + { + fieldErrors: { + tenantId: ['must contain only alphanumeric characters, hyphens, and underscores'], + }, + }, + ); + } + if (!/^[a-zA-Z0-9_-]+$/.test(runId)) { + throw new ValidationError( + 'Invalid run ID: must contain only alphanumeric characters, hyphens, and underscores', + { + fieldErrors: { + runId: ['must contain only alphanumeric characters, hyphens, and underscores'], + }, + }, + ); + } + this.namespace = getNamespace(); + } + + /** + * Creates a ConfigMap containing the provided files. + * Text files go in `data`, binary files go in `binaryData`. + */ + async initialize(files: Record): Promise { + if (this.isInitialized) { + throw new ConfigurationError('Volume already initialized', { + details: { configMapName: this.configMapName, tenantId: this.tenantId, runId: this.runId }, + }); + } + + const timestamp = Date.now(); + const tenantShort = sanitizeName(this.tenantId); + const runShort = sanitizeName(this.runId); + this.configMapName = `vol-${tenantShort}-${runShort}-${timestamp}`.slice(0, 63); + + try { + const data: Record = {}; + const binaryData: Record = {}; + + for (const [filename, content] of Object.entries(files)) { + this.validateFilename(filename); + + // ConfigMap keys can't have slashes — flatten paths + const key = filename.replace(/\//g, '__'); + + if (typeof content === 'string') { + data[key] = content; + } else { + // Buffer → base64 for binaryData + binaryData[key] = content.toString('base64'); + } + } + + const body: k8s.V1ConfigMap = { + metadata: { + name: this.configMapName, + namespace: this.namespace, + labels: { + 'app.kubernetes.io/managed-by': 'shipsec-worker', + 'shipsec.ai/purpose': 'isolated-volume', + 'shipsec.ai/tenant': tenantShort, + 'shipsec.ai/run': runShort, + }, + }, + data: Object.keys(data).length > 0 ? data : undefined, + binaryData: Object.keys(binaryData).length > 0 ? binaryData : undefined, + }; + + await getCoreApi().createNamespacedConfigMap({ + namespace: this.namespace, + body, + }); + + this.isInitialized = true; + return this.configMapName; + } catch (error) { + if (this.configMapName) { + await this.cleanup().catch(() => {}); + } + throw new ContainerError( + `Failed to initialize K8s volume: ${error instanceof Error ? error.message : String(error)}`, + { + cause: error instanceof Error ? error : undefined, + details: { tenantId: this.tenantId, runId: this.runId }, + }, + ); + } + } + + private validateFilename(filename: string): void { + if (filename.includes('..') || filename.startsWith('/')) { + throw new ValidationError(`Invalid filename (path traversal): ${filename}`, { + fieldErrors: { filename: ['path traversal not allowed'] }, + }); + } + const safePattern = /^[a-zA-Z0-9._/-]+$/; + if (!safePattern.test(filename)) { + throw new ValidationError(`Invalid filename (contains unsafe characters): ${filename}`, { + fieldErrors: { filename: ['contains unsafe characters'] }, + }); + } + } + + /** + * Read files from the ConfigMap. + */ + async readFiles(filenames: string[]): Promise> { + if (!this.configMapName) { + throw new ConfigurationError('Volume not initialized'); + } + + const cm = await getCoreApi().readNamespacedConfigMap({ + name: this.configMapName, + namespace: this.namespace, + }); + + const results: Record = {}; + for (const filename of filenames) { + const key = filename.replace(/\//g, '__'); + if (cm.data?.[key]) { + results[filename] = cm.data[key]; + } else if (cm.binaryData?.[key]) { + results[filename] = Buffer.from(cm.binaryData[key], 'base64').toString('utf-8'); + } + } + return results; + } + + /** + * Returns a bind mount string compatible with the K8s runner. + * Format: "configmap:::" + */ + getBindMount(containerPath = '/inputs', readOnly = true): string { + if (!this.configMapName) { + throw new ConfigurationError('Volume not initialized'); + } + const mode = readOnly ? 'ro' : 'rw'; + return `configmap:${this.configMapName}:${containerPath}:${mode}`; + } + + /** + * Returns volume config for the runner. The K8s runner recognizes the + * "configmap:" prefix in source and mounts the ConfigMap accordingly. + */ + getVolumeConfig(containerPath = '/inputs', readOnly = true) { + if (!this.configMapName) { + throw new ConfigurationError('Volume not initialized'); + } + return { + source: `configmap:${this.configMapName}`, + target: containerPath, + readOnly, + }; + } + + /** + * Delete the ConfigMap. + */ + async cleanup(): Promise { + if (!this.configMapName) return; + + try { + await getCoreApi().deleteNamespacedConfigMap({ + name: this.configMapName, + namespace: this.namespace, + }); + } catch (error) { + console.error( + `Failed to cleanup K8s volume ${this.configMapName}: ${error instanceof Error ? error.message : String(error)}`, + ); + } finally { + this.isInitialized = false; + this.configMapName = undefined; + } + } + + getVolumeName(): string | undefined { + return this.configMapName; + } +} From a7c88a163fd4d1dd7479539e3c4cc90794606cc1 Mon Sep 17 00:00:00 2001 From: betterclever Date: Thu, 12 Feb 2026 02:34:14 +0400 Subject: [PATCH 003/690] feat(release): add RBAC and release config for runtime Job execution --- .prettierignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.prettierignore b/.prettierignore index 373eb18fc..862421261 100644 --- a/.prettierignore +++ b/.prettierignore @@ -11,3 +11,6 @@ node_modules/ # Generated files *.generated.ts + +# Helm templates (Go template syntax is not valid YAML) +deploy/helm/*/templates/ From d53eb70948d547bd15dddcc034dde209357b3e84 Mon Sep 17 00:00:00 2001 From: betterclever Date: Thu, 12 Feb 2026 02:34:27 +0400 Subject: [PATCH 004/690] refactor(components): update components for runtime-compatible volume... --- .../components/ai/__tests__/opencode.test.ts | 16 +- worker/src/components/ai/opencode.ts | 19 +- .../src/components/core/mcp-group-runtime.ts | 356 ++++++++---------- worker/src/components/security/amass.ts | 61 +-- worker/src/components/security/dnsx.ts | 116 ++---- worker/src/components/security/httpx.ts | 49 +-- worker/src/components/security/nuclei.ts | 34 +- .../src/components/security/prowler-scan.ts | 96 ++--- .../components/security/shuffledns-massdns.ts | 41 +- worker/src/components/security/subfinder.ts | 54 +-- .../components/security/supabase-scanner.ts | 66 +--- worker/src/components/security/trufflehog.ts | 45 +-- worker/src/components/test/simple-http-mcp.ts | 4 +- .../activities/mcp-discovery.activity.ts | 2 + .../src/temporal/activities/mcp.activity.ts | 116 ++---- 15 files changed, 334 insertions(+), 741 deletions(-) diff --git a/worker/src/components/ai/__tests__/opencode.test.ts b/worker/src/components/ai/__tests__/opencode.test.ts index c14798390..70f1692c3 100644 --- a/worker/src/components/ai/__tests__/opencode.test.ts +++ b/worker/src/components/ai/__tests__/opencode.test.ts @@ -1,14 +1,14 @@ import { describe, it, expect, vi, beforeEach, afterAll } from 'bun:test'; import { componentRegistry } from '@shipsec/component-sdk'; import * as SDK from '@shipsec/component-sdk'; // Import for spying -import { IsolatedContainerVolume } from '../../../utils/isolated-volume'; +import { createIsolatedVolume } from '../../../utils/isolated-volume'; import * as utils from '../utils'; import '../opencode'; // Register the component -// Mock IsolatedContainerVolume +// Mock createIsolatedVolume vi.mock('../../../utils/isolated-volume', () => { return { - IsolatedContainerVolume: vi.fn().mockImplementation(() => ({ + createIsolatedVolume: vi.fn().mockImplementation(() => ({ initialize: vi.fn().mockResolvedValue('mock-volume-name'), cleanup: vi.fn().mockResolvedValue(undefined), getVolumeConfig: vi @@ -84,8 +84,8 @@ describe('shipsec.opencode.agent', () => { expect(result.report).toContain('# Report'); - expect(IsolatedContainerVolume).toHaveBeenCalled(); - const volumeInstance = (IsolatedContainerVolume as any).mock.results[0].value; + expect(createIsolatedVolume).toHaveBeenCalled(); + const volumeInstance = (createIsolatedVolume as any).mock.results[0].value; const initCall = volumeInstance.initialize.mock.calls[0][0]; expect(initCall['context.json']).toContain('"alertId": "123"'); @@ -93,7 +93,7 @@ describe('shipsec.opencode.agent', () => { expect(runSpy).toHaveBeenCalled(); const runnerCall = runSpy.mock.calls[0][0]; - expect(runnerCall.image).toBe('ghcr.io/shipsecai/opencode:latest'); + expect(runnerCall.image).toBe('ghcr.io/shipsecai/opencode:1.1.53'); expect(runnerCall.network).toBe('host'); expect(runnerCall.env.OPENAI_API_KEY).toBe('sk-test'); }); @@ -131,8 +131,8 @@ describe('shipsec.opencode.agent', () => { await component.execute({ inputs, params }, context as any); - expect(IsolatedContainerVolume).toHaveBeenCalled(); - const volumeInstance = (IsolatedContainerVolume as any).mock.results[0].value; + expect(createIsolatedVolume).toHaveBeenCalled(); + const volumeInstance = (createIsolatedVolume as any).mock.results[0].value; const initCall = volumeInstance.initialize.mock.calls[0][0]; const config = JSON.parse(initCall['opencode.jsonc']); diff --git a/worker/src/components/ai/opencode.ts b/worker/src/components/ai/opencode.ts index e83fd0ebe..e8b9fd2e9 100644 --- a/worker/src/components/ai/opencode.ts +++ b/worker/src/components/ai/opencode.ts @@ -11,7 +11,7 @@ import { param, } from '@shipsec/component-sdk'; import { LLMProviderSchema, llmProviderContractName } from '@shipsec/contracts'; -import { IsolatedContainerVolume } from '../../utils/isolated-volume'; +import { createIsolatedVolume } from '../../utils/isolated-volume'; import { DEFAULT_GATEWAY_URL, getGatewaySessionToken } from './utils'; const inputSchema = inputs({ @@ -99,7 +99,7 @@ const definition = defineComponent({ category: 'ai', runner: { kind: 'docker', - image: 'ghcr.io/shipsecai/opencode:latest', + image: 'ghcr.io/shipsecai/opencode:1.1.53', entrypoint: 'opencode', // We will override this in execution network: 'host' as const, // Required to access localhost gateway command: ['help'], @@ -241,14 +241,10 @@ Please investigate the issue and generate a detailed report. // 4. Setup Isolated Volume const tenantId = (context as any).tenantId ?? 'default-tenant'; - const volume = new IsolatedContainerVolume(tenantId, context.runId); + const volume = createIsolatedVolume(tenantId, context.runId); try { // 5. Execute Docker Container - // HACK: Fail fast after listing tools for faster iteration on MCP tool registration - // TODO: Remove this hack once MCP tool registration is working correctly - const HACK_FAIL_FAST_AFTER_TOOL_LIST = 'false'; - // Write a wrapper script to properly execute opencode with file reading // The script runs inside the container, so $(cat /workspace/prompt.txt) works correctly // Note: --quiet flag doesn't exist in opencode 1.1.34, use --log-level ERROR instead @@ -257,14 +253,7 @@ Please investigate the issue and generate a detailed report. 'set -e', 'cd /workspace', 'echo "[OpenCode] Listing MCP tools before run..."', - 'opencode mcp list --log-level ERROR > /tmp/mcp_tools.txt 2>&1', - 'cat /tmp/mcp_tools.txt', - 'echo "[OpenCode] === Full tool list output above ==="', - // HACK: Exit after listing tools for fast iteration - `if [ "${HACK_FAIL_FAST_AFTER_TOOL_LIST}" = "true" ]; then`, - ' echo "[OpenCode] HACK: Exiting after tool list for fast iteration"', - ' exit 1', - 'fi', + 'opencode mcp list --log-level ERROR || true', 'echo "[OpenCode] Starting agent run..."', 'opencode run --log-level ERROR "$(cat /workspace/prompt.txt)"', '', diff --git a/worker/src/components/core/mcp-group-runtime.ts b/worker/src/components/core/mcp-group-runtime.ts index 7ee16001a..e3b40ff60 100644 --- a/worker/src/components/core/mcp-group-runtime.ts +++ b/worker/src/components/core/mcp-group-runtime.ts @@ -1,9 +1,7 @@ import { z } from 'zod'; import type { ExecutionContext } from '@shipsec/component-sdk'; -import { Client } from '@modelcontextprotocol/sdk/client/index.js'; -import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js'; import { startMcpDockerServer } from './mcp-runtime'; -import { IsolatedContainerVolume } from '../../utils/isolated-volume'; +import { createIsolatedVolume } from '../../utils/isolated-volume'; /** * Schema for MCP Group Templates (code-defined) @@ -22,7 +20,6 @@ export const McpGroupTemplateSchema = z.object({ servers: z.array( z.object({ id: z.string(), - name: z.string(), command: z.string(), args: z.array(z.string()).optional(), }), @@ -53,6 +50,81 @@ export const GroupCredentialsSchema = z.object({ export type GroupCredentials = z.infer; +/** + * Fetches server details from the MCP Group Servers API + */ +async function fetchGroupServers( + groupSlug: string, + serverIds: string[], + context: ExecutionContext, +): Promise { + const backendUrl = process.env.BACKEND_URL || 'http://localhost:3000'; + const internalApiUrl = `${backendUrl}/internal/mcp`; + + // Generate internal API token + const tokenResponse = await fetch(`${internalApiUrl}/generate-token`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + runId: context.runId, + allowedNodeIds: [context.componentRef], + }), + }); + + if (!tokenResponse.ok) { + throw new Error(`Failed to generate internal API token: ${tokenResponse.statusText}`); + } + + const { token } = (await tokenResponse.json()) as { token: string }; + + const results: McpServerEndpoint[] = []; + + for (const serverId of serverIds) { + try { + const registerResponse = await fetch(`${internalApiUrl}/register-group-server`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${token}`, + }, + body: JSON.stringify({ + runId: context.runId, + nodeId: context.componentRef, + groupSlug, + serverId, + }), + }); + + if (!registerResponse.ok) { + throw new Error(`Failed to fetch server ${serverId}: ${registerResponse.statusText}`); + } + + const serverData = (await registerResponse.json()) as { + command: string; + args?: string[]; + endpoint?: string; + }; + + // For HTTP servers, return directly + if (serverData.endpoint) { + results.push({ + endpoint: serverData.endpoint, + containerId: '', + serverId, + }); + } + // For stdio servers, we'll start containers below + } catch (error) { + console.error(`Failed to fetch server ${serverId}:`, error); + throw error; + } + } + + return results; +} + /** * Maps credential contract values to environment variables * Supports both direct env mapping and AWS file generation @@ -133,44 +205,26 @@ export async function executeMcpGroupNode( params: { enabledServers: string[] }, groupTemplate: McpGroupTemplate, ): Promise<{ endpoints: McpServerEndpoint[] }> { - const enabledServers = params.enabledServers || []; - console.log(`[executeMcpGroupNode] ============================================`); - console.log(`[executeMcpGroupNode] Starting execution for group ${groupTemplate.slug}`); - console.log(`[executeMcpGroupNode] Component ref: ${context.componentRef}`); - console.log(`[executeMcpGroupNode] Run ID: ${context.runId}`); - console.log(`[executeMcpGroupNode] Enabled servers: ${enabledServers.join(', ')}`); - console.log( - `[executeMcpGroupNode] [DEBUG] componentRef should match workflow node ID for proper gateway filtering`, - ); - console.log( - `[executeMcpGroupNode] [DEBUG] Child server nodeIds will be: ${enabledServers.map((s) => `${context.componentRef}/${s}`).join(', ')}`, - ); - const credentials = inputs.credentials; + const enabledServers = params.enabledServers || []; if (!credentials || Object.keys(credentials).length === 0) { throw new Error('Credentials are required for MCP group execution'); } if (enabledServers.length === 0) { - console.log(`[executeMcpGroupNode] No enabled servers, returning empty endpoints`); return { endpoints: [] }; } // Build environment variables from credential mapping const env = buildCredentialEnv(credentials, groupTemplate.credentialMapping); - console.log(`[executeMcpGroupNode] Built credential env:`, Object.keys(env)); - // Get enabled servers from template (no API call needed!) - const enabledServerTemplates = groupTemplate.servers.filter((s) => enabledServers.includes(s.id)); - - console.log( - `[executeMcpGroupNode] Processing ${enabledServerTemplates.length} enabled servers from template`, - ); + // Fetch server details from backend + const serverDetails = await fetchGroupServers(groupTemplate.slug, enabledServers, context); const endpoints: McpServerEndpoint[] = []; - const volumes: ReturnType[] = []; - let volume: IsolatedContainerVolume | null = null; + const volumes: ReturnType['getVolumeConfig']>[] = []; + let volume: ReturnType | null = null; try { // Create volume if AWS files are needed @@ -178,7 +232,7 @@ export async function executeMcpGroupNode( const awsFiles = buildAwsCredentialFiles(credentials); if (awsFiles) { const tenantId = (context as any).tenantId ?? 'default-tenant'; - volume = new IsolatedContainerVolume(tenantId, context.runId); + volume = createIsolatedVolume(tenantId, context.runId); await volume.initialize({ credentials: awsFiles.credentials, config: awsFiles.config, @@ -187,76 +241,55 @@ export async function executeMcpGroupNode( } } - // Process each enabled server - for (const serverTemplate of enabledServerTemplates) { - console.log(`[executeMcpGroupNode] ----------------------------------------`); - console.log(`[executeMcpGroupNode] Starting container for server: ${serverTemplate.id}`); - console.log(`[executeMcpGroupNode] Command: ${serverTemplate.command}`); - console.log(`[executeMcpGroupNode] Args: ${JSON.stringify(serverTemplate.args || [])}`); - console.log(`[executeMcpGroupNode] Image: ${groupTemplate.defaultDockerImage}`); - - // Set MCP_COMMAND for the stdio proxy - // MCP_NAMED_SERVERS='{}' disables the built-in named-servers.json config - // so the proxy falls through to MCP_COMMAND mode - const serverEnv: Record = { - ...env, - MCP_COMMAND: serverTemplate.command, - MCP_NAMED_SERVERS: '{}', - }; + // Start container for each stdio server + for (const serverDetail of serverDetails) { + if (!serverDetail.endpoint) { + // This is a stdio server, need to start container + const serverTemplate = groupTemplate.servers.find((s) => s.id === serverDetail.serverId); + + if (!serverTemplate) { + throw new Error(`Server template not found: ${serverDetail.serverId}`); + } + + // Set MCP_COMMAND for the stdio proxy + const serverEnv: Record = { + ...env, + MCP_COMMAND: serverTemplate.command, + }; + + if (serverTemplate.args && serverTemplate.args.length > 0) { + serverEnv.MCP_ARGS = JSON.stringify(serverTemplate.args); + } + + const result = await startMcpDockerServer({ + image: groupTemplate.defaultDockerImage, + command: serverTemplate.command.split(' '), + env: serverEnv, + port: 0, // Auto-assign port + params: {}, + context, + volumes, + }); - if (serverTemplate.args && serverTemplate.args.length > 0) { - serverEnv.MCP_ARGS = JSON.stringify(serverTemplate.args); + // Register with backend + await registerServerWithBackend( + serverDetail.serverId, + result.endpoint, + result.containerId ?? '', + context, + ); + + endpoints.push({ + endpoint: result.endpoint, + containerId: result.containerId || '', + serverId: serverDetail.serverId, + }); + } else { + // HTTP server, already has endpoint + endpoints.push(serverDetail); } - - console.log(`[executeMcpGroupNode] Env vars:`, Object.keys(serverEnv)); - - const result = await startMcpDockerServer({ - image: groupTemplate.defaultDockerImage, - command: [], - env: serverEnv, - port: 0, // Auto-assign port - params: {}, - context, - volumes, - }); - - console.log(`[executeMcpGroupNode] Container started successfully!`); - console.log(`[executeMcpGroupNode] Endpoint: ${result.endpoint}`); - console.log(`[executeMcpGroupNode] Container ID: ${result.containerId}`); - - // Register with backend using hierarchical node ID (parent/child format) - // This allows explicit hierarchical queries instead of fragile prefix matching - const uniqueNodeId = `${context.componentRef}/${serverTemplate.id}`; - console.log(`[executeMcpGroupNode] Registering with backend...`); - console.log(`[executeMcpGroupNode] Unique nodeId: ${uniqueNodeId}`); - console.log( - `[executeMcpGroupNode] Backend URL: ${process.env.BACKEND_URL || 'http://localhost:3211'}`, - ); - - await registerServerWithBackend( - serverTemplate.id, - result.endpoint, - result.containerId ?? '', - context, - ); - - console.log(`[executeMcpGroupNode] Registration successful!`); - - endpoints.push({ - endpoint: result.endpoint, - containerId: result.containerId || '', - serverId: serverTemplate.id, - }); } - console.log(`[executeMcpGroupNode] ============================================`); - console.log(`[executeMcpGroupNode] Execution complete!`); - console.log(`[executeMcpGroupNode] Total endpoints: ${endpoints.length}`); - console.log( - `[executeMcpGroupNode] Endpoints:`, - endpoints.map((e) => `${e.serverId} -> ${e.endpoint}`), - ); - console.log(`[executeMcpGroupNode] ============================================`); return { endpoints }; } catch (error) { // Cleanup volume on error @@ -268,86 +301,7 @@ export async function executeMcpGroupNode( } /** - * Schema for discovered MCP tools - */ -interface McpTool { - name: string; - description?: string; - inputSchema?: Record; -} - -/** - * Discover tools from an MCP endpoint with exponential backoff retry. - * - * Uses the MCP SDK Client + StreamableHTTPClientTransport so that a proper - * `initialize` handshake is performed before `tools/list`. Many MCP servers - * (including the AWS MCP servers) reject a bare `tools/list` request without - * a preceding `initialize`, which caused the old raw-fetch implementation to - * silently return zero tools. - */ -async function discoverToolsWithRetry( - endpoint: string, - maxRetries = 8, - baseDelayMs = 1000, -): Promise { - let lastError: Error | null = null; - - for (let attempt = 1; attempt <= maxRetries; attempt++) { - let client: Client | null = null; - try { - console.log( - `[discoverToolsWithRetry] Attempt ${attempt}/${maxRetries}: Discovering tools from ${endpoint}`, - ); - - const transport = new StreamableHTTPClientTransport(new URL(endpoint), { - requestInit: { - headers: { - Accept: 'application/json, text/event-stream', - }, - }, - }); - - client = new Client( - { name: 'shipsec-worker-tool-discovery', version: '1.0.0' }, - { capabilities: {} }, - ); - - await client.connect(transport); - const res = await client.listTools(); - await client.close().catch(() => {}); - - const tools: McpTool[] = (res.tools ?? []).map((t) => ({ - name: t.name, - description: t.description, - inputSchema: t.inputSchema as Record | undefined, - })); - console.log( - `[discoverToolsWithRetry] ✓ Discovered ${tools.length} tools on attempt ${attempt}`, - ); - return tools; - } catch (error) { - lastError = error as Error; - await client?.close().catch(() => {}); - console.warn(`[discoverToolsWithRetry] Attempt ${attempt} failed: ${lastError.message}`); - - if (attempt < maxRetries) { - const delayMs = Math.min(baseDelayMs * Math.pow(2, attempt - 1), 5000); - console.log(`[discoverToolsWithRetry] Retrying in ${delayMs}ms...`); - await new Promise((resolve) => setTimeout(resolve, delayMs)); - } - } - } - - console.error( - `[discoverToolsWithRetry] ✗ Failed after ${maxRetries} attempts: ${lastError?.message}`, - ); - return []; -} - -/** - * Registers a server with the backend Tool Registry using the new clean API. - * - * Uses the /register-mcp-server endpoint which accepts pre-discovered tools. + * Registers a server with the backend Tool Registry */ async function registerServerWithBackend( serverId: string, @@ -355,49 +309,49 @@ async function registerServerWithBackend( containerId: string, context: ExecutionContext, ): Promise { - const backendUrl = process.env.BACKEND_URL || 'http://localhost:3211'; - const internalApiUrl = `${backendUrl}/api/v1/internal/mcp`; - const internalToken = process.env.INTERNAL_SERVICE_TOKEN || 'local-internal-token'; + const backendUrl = process.env.BACKEND_URL || 'http://localhost:3000'; + const internalApiUrl = `${backendUrl}/internal/mcp`; - // Use a unique nodeId for each server to avoid overwriting in Redis - // Format: ${groupNodeId}/${serverId} (e.g., "aws-mcp-group/aws-cloudtrail") - const uniqueNodeId = `${context.componentRef}/${serverId}`; + // Generate internal API token + const tokenResponse = await fetch(`${internalApiUrl}/generate-token`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + runId: context.runId, + allowedNodeIds: [context.componentRef], + }), + }); - console.log(`[registerServerWithBackend] Registering server ${serverId}`); - console.log(`[registerServerWithBackend] Unique nodeId: ${uniqueNodeId}`); - console.log(`[registerServerWithBackend] Endpoint: ${endpoint}`); + if (!tokenResponse.ok) { + throw new Error(`Failed to generate internal API token: ${tokenResponse.statusText}`); + } - // Discover tools from endpoint with retry logic - console.log(`[registerServerWithBackend] Discovering tools from endpoint...`); - const discoveredTools = await discoverToolsWithRetry(endpoint); - console.log(`[registerServerWithBackend] Discovered ${discoveredTools.length} tools`); + const { token } = (await tokenResponse.json()) as { token: string }; - // Register using the new clean API - const registerResponse = await fetch(`${internalApiUrl}/register-mcp-server`, { + // Register the local MCP with the Tool Registry + const registerResponse = await fetch(`${internalApiUrl}/register-local`, { method: 'POST', headers: { 'Content-Type': 'application/json', - 'x-internal-token': internalToken, + Authorization: `Bearer ${token}`, }, body: JSON.stringify({ runId: context.runId, - nodeId: uniqueNodeId, - serverName: serverId, - serverId, - transport: 'stdio', + nodeId: context.componentRef, + toolName: serverId, + description: `MCP tools from ${serverId}`, + inputSchema: { + type: 'object', + properties: {}, + }, endpoint, containerId, - tools: discoveredTools, }), }); if (!registerResponse.ok) { - const errorText = await registerResponse.text(); - console.error(`[registerServerWithBackend] Registration failed: ${errorText}`); throw new Error(`Failed to register server ${serverId}: ${registerResponse.statusText}`); } - - console.log( - `[registerServerWithBackend] ✓ Registered ${serverId} with ${discoveredTools.length} tools`, - ); } diff --git a/worker/src/components/security/amass.ts b/worker/src/components/security/amass.ts index 25bc9212c..3b88a27b8 100644 --- a/worker/src/components/security/amass.ts +++ b/worker/src/components/security/amass.ts @@ -11,15 +11,10 @@ import { port, param, type DockerRunnerConfig, - generateFindingHash, - analyticsResultSchema, - type AnalyticsResult, - type ExecutionContext, - type ExecutionPayload, } from '@shipsec/component-sdk'; -import { IsolatedContainerVolume } from '../../utils/isolated-volume'; +import { createIsolatedVolume } from '../../utils/isolated-volume'; -const AMASS_IMAGE = 'ghcr.io/shipsecai/amass:latest'; +const AMASS_IMAGE = 'ghcr.io/shipsecai/amass:v5.0.1'; const AMASS_TIMEOUT_SECONDS = (() => { const raw = process.env.AMASS_TIMEOUT_SECONDS; const parsed = raw ? Number.parseInt(raw, 10) : NaN; @@ -283,11 +278,6 @@ const outputSchema = outputs({ connectionType: { kind: 'primitive', name: 'json' }, }, ), - results: port(z.array(analyticsResultSchema()), { - label: 'Results', - description: - 'Analytics-ready findings with scanner, finding_hash, and severity. Connect to Analytics Sink.', - }), }); // Split custom CLI flags into an array of arguments @@ -462,7 +452,7 @@ const amassRetryPolicy: ComponentRetryPolicy = { nonRetryableErrorTypes: ['ContainerError', 'ValidationError', 'ConfigurationError'], }; -const definition = (defineComponent as any)({ +const definition = defineComponent({ id: 'shipsec.amass.enum', label: 'Amass Enumeration', category: 'security', @@ -470,24 +460,24 @@ const definition = (defineComponent as any)({ runner: { kind: 'docker', image: AMASS_IMAGE, - // The amass image is distroless (no shell available). - // Use the image's default entrypoint directly and pass args via command. + // IMPORTANT: Use shell wrapper for PTY compatibility + // Running CLI tools directly as entrypoint can cause them to hang with PTY (pseudo-terminal) + // The shell wrapper ensures proper TTY signal handling and clean exit + // See docs/component-development.md "Docker Entrypoint Pattern" for details + entrypoint: 'sh', network: 'bridge', timeoutSeconds: AMASS_TIMEOUT_SECONDS, env: { HOME: '/tmp', }, - command: [], + // Shell wrapper pattern: sh -c 'amass "$@"' -- [args...] + // This allows dynamic args to be appended and properly passed to amass + command: ['-c', 'amass "$@"', '--'], }, inputs: inputSchema, outputs: outputSchema, parameters: parameterSchema, docs: 'Enumerate subdomains with OWASP Amass. Supports active techniques, brute forcing, alterations, recursion tuning, and DNS throttling.', - toolProvider: { - kind: 'component', - name: 'amass_enum', - description: 'Deep subdomain enumeration and attack surface mapping tool (Amass).', - }, ui: { slug: 'amass', version: '1.0.0', @@ -505,6 +495,10 @@ const definition = (defineComponent as any)({ }, isLatest: true, deprecated: false, + agentTool: { + enabled: true, + toolDescription: 'Deep subdomain enumeration and attack surface mapping tool (Amass).', + }, example: '`amass enum -d example.com -brute -alts` - Aggressively enumerates subdomains with brute force and alteration engines enabled.', examples: [ @@ -512,13 +506,7 @@ const definition = (defineComponent as any)({ 'Perform quick passive reconnaissance using custom CLI flags like --passive.', ], }, - async execute( - { - inputs, - params, - }: ExecutionPayload, z.infer>, - context: ExecutionContext, - ) { + async execute({ inputs, params }, context) { const parsedParams = parameterSchema.parse(params); const { passive, @@ -595,7 +583,7 @@ const definition = (defineComponent as any)({ const tenantId = (context as any).tenantId ?? 'default-tenant'; // Create isolated volume for this execution - const volume = new IsolatedContainerVolume(tenantId, context.runId); + const volume = createIsolatedVolume(tenantId, context.runId); const baseRunner = definition.runner; if (baseRunner.kind !== 'docker') { @@ -638,7 +626,9 @@ const definition = (defineComponent as any)({ network: baseRunner.network, timeoutSeconds: baseRunner.timeoutSeconds ?? AMASS_TIMEOUT_SECONDS, env: { ...(baseRunner.env ?? {}) }, - // Pass amass CLI args directly (image default entrypoint is amass) + // Preserve the shell wrapper from baseRunner (sh -c 'amass "$@"' --) + entrypoint: baseRunner.entrypoint, + // Append amass arguments to shell wrapper command command: [...(baseRunner.command ?? []), ...amassArgs], volumes: [volume.getVolumeConfig(CONTAINER_INPUT_DIR, true)], }; @@ -728,23 +718,12 @@ const definition = (defineComponent as any)({ }); } - // Build analytics-ready results with scanner metadata - const analyticsResults: AnalyticsResult[] = subdomains.map((subdomain) => ({ - scanner: 'amass', - finding_hash: generateFindingHash('subdomain-discovery', subdomain, inputs.domains.join(',')), - severity: 'info' as const, - asset_key: subdomain, - subdomain, - parent_domains: inputs.domains, - })); - return { subdomains, rawOutput, domainCount, subdomainCount, options: optionsSummary, - results: analyticsResults, }; }, }); diff --git a/worker/src/components/security/dnsx.ts b/worker/src/components/security/dnsx.ts index 55eb41d97..cf7a53051 100644 --- a/worker/src/components/security/dnsx.ts +++ b/worker/src/components/security/dnsx.ts @@ -11,11 +11,8 @@ import { parameters, port, param, - generateFindingHash, - analyticsResultSchema, - type AnalyticsResult, } from '@shipsec/component-sdk'; -import { IsolatedContainerVolume } from '../../utils/isolated-volume'; +import { createIsolatedVolume } from '../../utils/isolated-volume'; const recordTypeEnum = z.enum([ 'A', @@ -35,7 +32,7 @@ const recordTypeEnum = z.enum([ const outputModeEnum = z.enum(['silent', 'json']); -const DNSX_IMAGE = 'ghcr.io/shipsecai/dnsx:latest'; +const DNSX_IMAGE = 'ghcr.io/shipsecai/dnsx:v1.2.2'; const DNSX_TIMEOUT_SECONDS = 180; const INPUT_MOUNT_NAME = 'inputs'; const CONTAINER_INPUT_DIR = `/${INPUT_MOUNT_NAME}`; @@ -238,8 +235,8 @@ const dnsxLineSchema = z .passthrough(); const outputSchema = outputs({ - dnsRecords: port(z.array(z.any()), { - label: 'DNS Records', + results: port(z.array(z.any()), { + label: 'Results', description: 'DNS resolution results returned by dnsx.', allowAny: true, reason: 'dnsx returns heterogeneous record payloads.', @@ -273,11 +270,6 @@ const outputSchema = outputs({ label: 'Errors', description: 'Errors encountered during dnsx execution.', }), - results: port(z.array(analyticsResultSchema()), { - label: 'Results', - description: - 'Analytics-ready findings with scanner, finding_hash, and severity. Connect to Analytics Sink.', - }), }); const splitCliArgs = (input: string): string[] => { @@ -483,26 +475,24 @@ const definition = defineComponent({ runner: { kind: 'docker', image: DNSX_IMAGE, - // The dnsx image is distroless (no shell available). - // Use the image's default entrypoint directly and pass args via command. + // IMPORTANT: Use shell wrapper for PTY compatibility + // Running CLI tools directly as entrypoint can cause them to hang with PTY (pseudo-terminal) + // The shell wrapper ensures proper TTY signal handling and clean exit + // See docs/component-development.md "Docker Entrypoint Pattern" for details + entrypoint: 'sh', network: 'bridge', timeoutSeconds: DNSX_TIMEOUT_SECONDS, env: { - // Image runs as nonroot — /root is not writable. - // Use /tmp so dnsx can create its config dir. - HOME: '/tmp', + HOME: '/root', }, - command: [], + // Shell wrapper pattern: sh -c 'dnsx "$@"' -- [args...] + // This allows dynamic args to be appended and properly passed to dnsx + command: ['-c', 'dnsx "$@"', '--'], }, inputs: inputSchema, outputs: outputSchema, parameters: parameterSchema, docs: 'Executes dnsx inside Docker to resolve DNS records for the provided domains. Supports multiple record types, custom resolvers, and rate limiting.', - toolProvider: { - kind: 'component', - name: 'dns_resolver', - description: 'DNS resolution and record lookup tool (dnsx).', - }, ui: { slug: 'dnsx', version: '1.0.0', @@ -518,6 +508,10 @@ const definition = defineComponent({ }, isLatest: true, deprecated: false, + agentTool: { + enabled: true, + toolDescription: 'DNS resolution and record lookup tool (dnsx).', + }, }, async execute({ inputs, params }, context) { const parsedParams = parameterSchema.parse(params); @@ -572,7 +566,6 @@ const definition = defineComponent({ if (domainCount === 0) { context.logger.info('[DNSX] Skipping dnsx execution because no domains were provided.'); return outputSchema.parse({ - dnsRecords: [], results: [], rawOutput: '', domainCount: 0, @@ -621,7 +614,7 @@ const definition = defineComponent({ const tenantId = (context as any).tenantId ?? 'default-tenant'; // Create isolated volume for this execution - const volume = new IsolatedContainerVolume(tenantId, context.runId); + const volume = createIsolatedVolume(tenantId, context.runId); const baseRunner = definition.runner; if (baseRunner.kind !== 'docker') { @@ -652,7 +645,11 @@ const definition = defineComponent({ network: baseRunner.network, timeoutSeconds: baseRunner.timeoutSeconds ?? DNSX_TIMEOUT_SECONDS, env: { ...(baseRunner.env ?? {}) }, - // Pass dnsx CLI args directly (image default entrypoint is dnsx) + // Preserve the shell wrapper from baseRunner (sh -c 'dnsx "$@"' --) + // This is critical for PTY compatibility - do not override with 'dnsx' + entrypoint: baseRunner.entrypoint, + // Append dnsx arguments to shell wrapper command + // Resulting command: ['sh', '-c', 'dnsx "$@"', '--', ...dnsxArgs] command: [...(baseRunner.command ?? []), ...dnsxArgs], volumes: [volume.getVolumeConfig(CONTAINER_INPUT_DIR, true)], }; @@ -773,24 +770,8 @@ const definition = defineComponent({ .filter((host): host is string => typeof host === 'string' && host.length > 0), ); - // Build analytics-ready results with scanner metadata - const analyticsResults: AnalyticsResult[] = normalisedRecords.map((record) => ({ - scanner: 'dnsx', - finding_hash: generateFindingHash( - 'dns-resolution', - record.host, - JSON.stringify(record.answers), - ), - severity: 'info' as const, - asset_key: record.host, - host: record.host, - record_types: Object.keys(record.answers), - answers: record.answers, - })); - return { - dnsRecords: normalisedRecords, - results: analyticsResults, + results: normalisedRecords, rawOutput: params.rawOutput, domainCount: params.domainCount, recordCount: params.recordCount, @@ -829,7 +810,6 @@ const definition = defineComponent({ if (trimmed.length === 0) { return { - dnsRecords: [], results: [], rawOutput, domainCount: domainCount, @@ -854,7 +834,6 @@ const definition = defineComponent({ ? (record.domainCount as number) : domainCount; return { - dnsRecords: [], results: [], rawOutput: trimmed, domainCount: errorDomainCount, @@ -869,10 +848,10 @@ const definition = defineComponent({ const validated = outputSchema.safeParse(record); if (validated.success) { return buildOutput({ - records: validated.data.dnsRecords as z.infer[], + records: validated.data.results as z.infer[], rawOutput: validated.data.rawOutput ?? rawOutput, domainCount: validated.data.domainCount ?? domainCount, - recordCount: validated.data.recordCount ?? validated.data.dnsRecords.length, + recordCount: validated.data.recordCount ?? validated.data.results.length, recordTypes: validated.data.recordTypes ?? recordTypes, resolvers: validated.data.resolvers ?? resolverList, errors: validated.data.errors, @@ -890,7 +869,6 @@ const definition = defineComponent({ if (lines.length === 0) { return { - dnsRecords: [], results: [], rawOutput, domainCount: domainCount, @@ -917,24 +895,8 @@ const definition = defineComponent({ }; }); - // Build analytics-ready results - const analyticsResults: AnalyticsResult[] = silentRecords.map((record) => ({ - scanner: 'dnsx', - finding_hash: generateFindingHash( - 'dns-resolution', - record.host, - JSON.stringify(record.answers), - ), - severity: 'info' as const, - asset_key: record.host, - host: record.host, - record_types: Object.keys(record.answers), - answers: record.answers, - })); - return { - dnsRecords: silentRecords, - results: analyticsResults, + results: silentRecords, rawOutput, domainCount: domainCount, recordCount: silentRecords.length, @@ -957,7 +919,6 @@ const definition = defineComponent({ if (trimmed.length === 0) { return { - dnsRecords: [], results: [], rawOutput, domainCount: domainCount, @@ -1014,24 +975,8 @@ const definition = defineComponent({ }; }); - // Build analytics-ready results - const analyticsResults: AnalyticsResult[] = fallbackResults.map((record) => ({ - scanner: 'dnsx', - finding_hash: generateFindingHash( - 'dns-resolution', - record.host, - JSON.stringify(record.answers), - ), - severity: 'info' as const, - asset_key: record.host, - host: record.host, - record_types: Object.keys(record.answers), - answers: record.answers, - })); - return { - dnsRecords: fallbackResults, - results: analyticsResults, + results: fallbackResults, rawOutput, domainCount: domainCount, recordCount: fallbackResults.length, @@ -1071,7 +1016,6 @@ const definition = defineComponent({ : JSON.stringify(rawPayload, null, 2).slice(0, 5000); return { - dnsRecords: [], results: [], rawOutput, domainCount: domainCount, @@ -1084,10 +1028,10 @@ const definition = defineComponent({ } return buildOutput({ - records: safeResult.data.dnsRecords as z.infer[], + records: safeResult.data.results as z.infer[], rawOutput: safeResult.data.rawOutput, domainCount: safeResult.data.domainCount ?? domainCount, - recordCount: safeResult.data.recordCount ?? safeResult.data.dnsRecords.length, + recordCount: safeResult.data.recordCount ?? safeResult.data.results.length, recordTypes: safeResult.data.recordTypes, resolvers: safeResult.data.resolvers, errors: safeResult.data.errors, diff --git a/worker/src/components/security/httpx.ts b/worker/src/components/security/httpx.ts index ee95318e4..a36085743 100644 --- a/worker/src/components/security/httpx.ts +++ b/worker/src/components/security/httpx.ts @@ -10,11 +10,8 @@ import { parameters, port, param, - generateFindingHash, - analyticsResultSchema, - type AnalyticsResult, } from '@shipsec/component-sdk'; -import { IsolatedContainerVolume } from '../../utils/isolated-volume'; +import { createIsolatedVolume } from '../../utils/isolated-volume'; const inputSchema = inputs({ targets: port( @@ -148,7 +145,7 @@ const findingSchema = z.object({ type Finding = z.infer; const outputSchema = outputs({ - responses: port(z.array(findingSchema), { + results: port(z.array(findingSchema), { label: 'HTTP Responses', description: 'Structured metadata for each responsive endpoint.', connectionType: { kind: 'list', element: { kind: 'primitive', name: 'json' } }, @@ -181,11 +178,6 @@ const outputSchema = outputs({ connectionType: { kind: 'primitive', name: 'json' }, }, ), - results: port(z.array(analyticsResultSchema()), { - label: 'Results', - description: - 'Analytics-ready findings with scanner, finding_hash, and severity. Connect to Analytics Sink.', - }), }); const httpxRunnerOutputSchema = z.object({ @@ -210,7 +202,7 @@ const definition = defineComponent({ category: 'security', runner: { kind: 'docker', - image: 'ghcr.io/shipsecai/httpx:latest', + image: 'ghcr.io/shipsecai/httpx:v1.7.4', entrypoint: 'httpx', network: 'bridge', timeoutSeconds: dockerTimeoutSeconds, @@ -247,15 +239,16 @@ const definition = defineComponent({ }, isLatest: true, deprecated: false, + example: + '`httpx -l targets.txt -json -status-code 200,301` - Probe discovered hosts and capture responsive endpoints with matching status codes.', examples: [ 'Validate Subfinder or Amass discoveries by probing for live web services.', 'Filter Naabu results to identify hosts exposing HTTP/S services on uncommon ports.', ], - }, - toolProvider: { - kind: 'component', - name: 'httpx_probe', - description: 'Live HTTP endpoint probe and metadata collector (httpx).', + agentTool: { + enabled: true, + toolDescription: 'Live HTTP endpoint probe and metadata collector (httpx).', + }, }, async execute({ inputs, params }, context) { const parsedParams = parameterSchema.parse(params); @@ -279,7 +272,6 @@ const definition = defineComponent({ if (runnerParams.targets.length === 0) { context.logger.info('[httpx] Skipping httpx probe because no targets were provided.'); const emptyOutput: Output = { - responses: [], results: [], rawOutput: '', targetCount: 0, @@ -309,7 +301,7 @@ const definition = defineComponent({ }); const tenantId = (context as any).tenantId ?? 'default-tenant'; - const volume = new IsolatedContainerVolume(tenantId, context.runId); + const volume = createIsolatedVolume(tenantId, context.runId); try { const targets = Array.from( @@ -403,27 +395,8 @@ const definition = defineComponent({ `[httpx] Completed probe with ${findings.length} result(s) from ${runnerParams.targets.length} target(s)`, ); - // Build analytics-ready results with scanner metadata - const analyticsResults: AnalyticsResult[] = findings.map((finding) => ({ - scanner: 'httpx', - finding_hash: generateFindingHash( - 'http-endpoint', - finding.url, - String(finding.statusCode ?? 0), - ), - severity: 'info' as const, - asset_key: finding.url, - url: finding.url, - host: finding.host, - status_code: finding.statusCode, - title: finding.title, - webserver: finding.webserver, - technologies: finding.technologies, - })); - const output: Output = { - responses: findings, - results: analyticsResults, + results: findings, rawOutput: runnerOutput, targetCount: runnerParams.targets.length, resultCount: findings.length, diff --git a/worker/src/components/security/nuclei.ts b/worker/src/components/security/nuclei.ts index c9d378d2f..4e576752c 100644 --- a/worker/src/components/security/nuclei.ts +++ b/worker/src/components/security/nuclei.ts @@ -12,11 +12,8 @@ import { parameters, port, param, - generateFindingHash, - analyticsResultSchema, - type AnalyticsResult, } from '@shipsec/component-sdk'; -import { IsolatedContainerVolume } from '../../utils/isolated-volume'; +import { createIsolatedVolume } from '../../utils/isolated-volume'; import * as yaml from 'js-yaml'; const inputSchema = inputs({ @@ -206,11 +203,6 @@ const outputSchema = outputs({ description: 'Array of detected vulnerabilities with severity, tags, and matched URLs.', connectionType: { kind: 'list', element: { kind: 'primitive', name: 'json' } }, }), - results: port(z.array(analyticsResultSchema()), { - label: 'Results', - description: - 'Analytics-ready findings with scanner, finding_hash, and severity. Connect to Analytics Sink.', - }), rawOutput: port(z.string(), { label: 'Raw Output', description: 'Complete JSONL output from nuclei for downstream processing.', @@ -291,12 +283,6 @@ const definition = defineComponent({ outputs: outputSchema, parameters: parameterSchema, docs: 'Run ProjectDiscovery Nuclei vulnerability scanner with custom or built-in templates. Supports quick YAML testing or bulk scans with template archives.', - toolProvider: { - kind: 'component', - name: 'nuclei_scan', - description: - 'Fast vulnerability scanner for CVEs, misconfigurations, and exposures using YAML templates.', - }, ui: { slug: 'nuclei', version: '1.0.0', @@ -322,6 +308,11 @@ const definition = defineComponent({ 'Bulk custom scan: Upload zip archive via Entry Point → File Loader → Nuclei', 'Comprehensive scan: Combine custom archive + built-in templates for complete coverage', ], + agentTool: { + enabled: true, + toolDescription: + 'Fast vulnerability scanner for CVEs, misconfigurations, and exposures using YAML templates.', + }, }, async execute({ inputs, params }, context) { const parsedInputs = inputSchema.parse(inputs); @@ -330,7 +321,7 @@ const definition = defineComponent({ context.logger.info(`[Nuclei] Starting scan for ${parsedInputs.targets.length} target(s)`); const tenantId = (context as any).tenantId ?? 'default-tenant'; - let volume: IsolatedContainerVolume | null = null; + let volume: ReturnType | null = null; try { const hasCustomArchive = !!parsedInputs.customTemplateArchive; @@ -393,7 +384,7 @@ const definition = defineComponent({ } // ===== TypeScript: Prepare all files for volume ===== - volume = new IsolatedContainerVolume(tenantId, context.runId); + volume = createIsolatedVolume(tenantId, context.runId); const files: Record = {}; // Always add targets file @@ -558,17 +549,8 @@ const definition = defineComponent({ `[Nuclei] Scan complete: ${findings.length} finding(s) from ${parsedInputs.targets.length} target(s)`, ); - // Build analytics-ready results with scanner metadata (follows core.analytics.result.v1 contract) - const results: AnalyticsResult[] = findings.map((finding) => ({ - ...finding, - scanner: 'nuclei', - asset_key: finding.host ?? finding.matchedAt, - finding_hash: generateFindingHash(finding.templateId, finding.host, finding.matchedAt), - })); - const output = { findings, - results, rawOutput: stdout, targetCount: parsedInputs.targets.length, findingCount: findings.length, diff --git a/worker/src/components/security/prowler-scan.ts b/worker/src/components/security/prowler-scan.ts index 78077771a..fa53c3c8e 100644 --- a/worker/src/components/security/prowler-scan.ts +++ b/worker/src/components/security/prowler-scan.ts @@ -14,14 +14,11 @@ import { parameters, port, param, - analyticsResultSchema, - generateFindingHash, - type AnalyticsResult, } from '@shipsec/component-sdk'; import type { DockerRunnerConfig } from '@shipsec/component-sdk'; import { awsCredentialSchema } from '@shipsec/contracts'; -import { IsolatedContainerVolume } from '../../utils/isolated-volume'; +import { createIsolatedVolume } from '../../utils/isolated-volume'; const recommendedFlagOptions = [ { @@ -250,11 +247,6 @@ const outputSchema = outputs({ 'Array of normalized findings derived from Prowler ASFF output (includes severity, resource id, remediation).', connectionType: { kind: 'list', element: { kind: 'primitive', name: 'json' } }, }), - results: port(z.array(analyticsResultSchema()), { - label: 'Results', - description: - 'Analytics-ready findings with scanner, finding_hash, and severity. Connect to Analytics Sink.', - }), rawOutput: port(z.string(), { label: 'Raw Output', description: 'Raw Prowler output for debugging.', @@ -296,10 +288,27 @@ const recommendedFlagMap = new Map( recommendedFlagOptions.map((option) => [option.id, [...option.args]]), ); -async function listVolumeFiles(volume: IsolatedContainerVolume): Promise { +async function listVolumeFiles(volume: ReturnType): Promise { const volumeName = volume.getVolumeName(); if (!volumeName) return []; + // In K8s mode, volumes are ConfigMap-backed — list keys via K8s API + if (process.env.EXECUTION_MODE === 'k8s') { + try { + const k8s = await import('@kubernetes/client-node'); + const kc = new k8s.KubeConfig(); + kc.loadFromCluster(); + const coreApi = kc.makeApiClient(k8s.CoreV1Api); + const namespace = process.env.K8S_JOB_NAMESPACE || 'shipsec-workloads'; + const cm = await coreApi.readNamespacedConfigMap({ name: volumeName, namespace }); + const keys = [...Object.keys(cm.data || {}), ...Object.keys(cm.binaryData || {})]; + // Unflatten __ back to / (ConfigMap key encoding from IsolatedK8sVolume) + return keys.map((k) => k.replace(/__/g, '/')); + } catch { + return []; + } + } + const dockerPath = await resolveDockerPath(); return new Promise((resolve, reject) => { const proc = spawn(dockerPath, [ @@ -350,13 +359,16 @@ async function listVolumeFiles(volume: IsolatedContainerVolume): Promise, uid = 1000, gid = 1000, ): Promise { const volumeName = volume.getVolumeName(); if (!volumeName) return; + // ConfigMap volumes in K8s are read-only projections — ownership is N/A + if (process.env.EXECUTION_MODE === 'k8s') return; + const dockerPath = await resolveDockerPath(); return new Promise((resolve, reject) => { const proc = spawn(dockerPath, [ @@ -407,7 +419,7 @@ const definition = defineComponent({ retryPolicy: prowlerRetryPolicy, runner: { kind: 'docker', - image: 'ghcr.io/shipsecai/prowler:latest', + image: 'ghcr.io/shipsecai/prowler:5.14.2', platform: 'linux/amd64', command: [], // Placeholder - actual command built dynamically in execute() }, @@ -415,11 +427,6 @@ const definition = defineComponent({ outputs: outputSchema, parameters: parameterSchema, docs: 'Execute Prowler inside Docker using `ghcr.io/shipsecai/prowler` (amd64 enforced on ARM hosts). Supports AWS account scans and the multi-cloud `prowler cloud` overview, with optional CLI flag customisation.', - toolProvider: { - kind: 'component', - name: 'prowler_scan', - description: 'AWS and multi-cloud security assessment tool (Prowler).', - }, ui: { slug: 'prowler-scan', version: '2.0.0', @@ -439,6 +446,10 @@ const definition = defineComponent({ 'Run nightly `prowler aws --quick --severity-filter high,critical` scans on production accounts and forward findings into ELK.', 'Use `prowler cloud` with custom flags to generate a multi-cloud compliance snapshot.', ], + agentTool: { + enabled: true, + toolDescription: 'AWS and multi-cloud security assessment tool (Prowler).', + }, }, async execute({ inputs, params }, context) { const parsedInputs = inputSchema.parse(inputs); @@ -509,7 +520,7 @@ const definition = defineComponent({ const awsEnv: Record = {}; const tenantId = (context as any).tenantId ?? 'default-tenant'; const awsCredsVolume = parsedInputs.credentials - ? new IsolatedContainerVolume(tenantId, `${context.runId}-prowler-aws`) + ? createIsolatedVolume(tenantId, `${context.runId}-prowler-aws`) : null; if (parsedInputs.credentials) { @@ -575,7 +586,7 @@ const definition = defineComponent({ // Prepare a one-off runner with dynamic command and volume const dockerRunner: DockerRunnerConfig = { kind: 'docker', - image: 'ghcr.io/shipsecai/prowler:latest', + image: 'ghcr.io/shipsecai/prowler:5.14.2', platform: 'linux/amd64', network: 'bridge', timeoutSeconds: 900, @@ -590,7 +601,7 @@ const definition = defineComponent({ let rawSegments: string[] = []; let commandForOutput: string[] = cmd; let stderrCombined = ''; - const outputVolume = new IsolatedContainerVolume(tenantId, `${context.runId}-prowler-out`); + const outputVolume = createIsolatedVolume(tenantId, `${context.runId}-prowler-out`); let outputVolumeInitialized = false; let awsVolumeInitialized = false; @@ -743,29 +754,9 @@ const definition = defineComponent({ const scanId = buildScanId(parsedInputs.accountId, parsedParams.scanMode); - // Build analytics-ready results (follows core.analytics.result.v1 contract) - const results: AnalyticsResult[] = findings.map((finding) => ({ - scanner: 'prowler', - finding_hash: generateFindingHash( - finding.id, - finding.resourceId ?? finding.accountId ?? '', - finding.title ?? '', - ), - severity: mapToAnalyticsSeverity(finding.severity), - asset_key: finding.resourceId ?? finding.accountId ?? undefined, - // Include additional context for analytics - title: finding.title, - description: finding.description, - region: finding.region, - status: finding.status, - remediationText: finding.remediationText, - recommendationUrl: finding.recommendationUrl, - })); - const output: Output = { scanId, findings, - results, rawOutput: rawSegments.join('\n'), summary: { totalFindings: findings.length, @@ -972,31 +963,6 @@ function extractRegionFromArn(resourceId?: string): string | null { return null; } -/** - * Maps Prowler severity levels to analytics severity enum. - * Prowler: critical, high, medium, low, informational, unknown - * Analytics: critical, high, medium, low, info, none - */ -function mapToAnalyticsSeverity( - prowlerSeverity: NormalisedSeverity, -): 'critical' | 'high' | 'medium' | 'low' | 'info' | 'none' { - switch (prowlerSeverity) { - case 'critical': - return 'critical'; - case 'high': - return 'high'; - case 'medium': - return 'medium'; - case 'low': - return 'low'; - case 'informational': - return 'info'; - case 'unknown': - default: - return 'none'; - } -} - componentRegistry.register(definition); // Create local type aliases for backward compatibility diff --git a/worker/src/components/security/shuffledns-massdns.ts b/worker/src/components/security/shuffledns-massdns.ts index d3690beab..f9ab23455 100644 --- a/worker/src/components/security/shuffledns-massdns.ts +++ b/worker/src/components/security/shuffledns-massdns.ts @@ -12,11 +12,8 @@ import { parameters, port, param, - generateFindingHash, - analyticsResultSchema, - type AnalyticsResult, } from '@shipsec/component-sdk'; -import { IsolatedContainerVolume } from '../../utils/isolated-volume'; +import { createIsolatedVolume } from '../../utils/isolated-volume'; const DEFAULT_RESOLVERS = ['1.1.1.1', '8.8.8.8'] as const; @@ -165,11 +162,6 @@ const outputSchema = outputs({ label: 'Subdomain Count', description: 'Number of unique subdomains discovered.', }), - results: port(z.array(analyticsResultSchema()), { - label: 'Results', - description: - 'Analytics-ready findings with scanner, finding_hash, and severity. Connect to Analytics Sink.', - }), }); const definition = defineComponent({ @@ -288,7 +280,7 @@ const definition = defineComponent({ // Write lists to an isolated volume and mount into the container const tenantId = (context as any).tenantId ?? 'default-tenant'; - const volume = new IsolatedContainerVolume(tenantId, context.runId); + const volume = createIsolatedVolume(tenantId, context.runId); const WORDS = 'words.txt'; const SEEDS = 'seeds.txt'; const RESOLVERS = 'resolvers.txt'; @@ -379,19 +371,8 @@ const definition = defineComponent({ const deduped = Array.from(new Set(subdomains)); - // Build analytics-ready results with scanner metadata - const analyticsResults: AnalyticsResult[] = deduped.map((subdomain) => ({ - scanner: 'shuffledns', - finding_hash: generateFindingHash('subdomain-discovery', subdomain, domains.join(',')), - severity: 'info' as const, - asset_key: subdomain, - subdomain, - parent_domains: domains, - })); - return outputSchema.parse({ subdomains: deduped, - results: analyticsResults, rawOutput, domainCount: domains.length, subdomainCount: deduped.length, @@ -416,31 +397,17 @@ const definition = defineComponent({ .map((line) => line.trim()) .filter((line) => line.length > 0); - const deduped = Array.from(new Set(subdomainsValue)); - - // Build analytics-ready results - const analyticsResults: AnalyticsResult[] = deduped.map((subdomain) => ({ - scanner: 'shuffledns', - finding_hash: generateFindingHash('subdomain-discovery', subdomain, domains.join(',')), - severity: 'info' as const, - asset_key: subdomain, - subdomain, - parent_domains: domains, - })); - return outputSchema.parse({ - subdomains: deduped, - results: analyticsResults, + subdomains: Array.from(new Set(subdomainsValue)), rawOutput: maybeRaw || subdomainsValue.join('\n'), domainCount: domains.length, - subdomainCount: deduped.length, + subdomainCount: subdomainsValue.length, }); } // Fallback – empty return outputSchema.parse({ subdomains: [], - results: [], rawOutput: '', domainCount: domains.length, subdomainCount: 0, diff --git a/worker/src/components/security/subfinder.ts b/worker/src/components/security/subfinder.ts index e58c26aac..e0b9a1f0d 100644 --- a/worker/src/components/security/subfinder.ts +++ b/worker/src/components/security/subfinder.ts @@ -11,13 +11,10 @@ import { parameters, port, param, - generateFindingHash, - analyticsResultSchema, - type AnalyticsResult, } from '@shipsec/component-sdk'; -import { IsolatedContainerVolume } from '../../utils/isolated-volume'; +import { createIsolatedVolume } from '../../utils/isolated-volume'; -const SUBFINDER_IMAGE = 'ghcr.io/shipsecai/subfinder:latest'; +const SUBFINDER_IMAGE = 'ghcr.io/shipsecai/subfinder:v2.12.0'; const SUBFINDER_TIMEOUT_SECONDS = 1800; // 30 minutes const INPUT_MOUNT_NAME = 'inputs'; const CONTAINER_INPUT_DIR = `/${INPUT_MOUNT_NAME}`; @@ -126,11 +123,6 @@ const outputSchema = outputs({ label: 'Subdomain Count', description: 'Number of subdomains discovered.', }), - results: port(z.array(analyticsResultSchema()), { - label: 'Results', - description: - 'Analytics-ready findings with scanner, finding_hash, and severity. Connect to Analytics Sink.', - }), }); // Split custom CLI flags into an array of arguments @@ -271,16 +263,19 @@ const definition = defineComponent({ runner: { kind: 'docker', image: SUBFINDER_IMAGE, - // The subfinder image is distroless (no shell available). - // Use the image's default entrypoint directly and pass args via command. + // IMPORTANT: Use shell wrapper for PTY compatibility + // Running CLI tools directly as entrypoint can cause them to hang with PTY (pseudo-terminal) + // The shell wrapper ensures proper TTY signal handling and clean exit + // See docs/component-development.md "Docker Entrypoint Pattern" for details + entrypoint: 'sh', network: 'bridge', timeoutSeconds: SUBFINDER_TIMEOUT_SECONDS, env: { - // Image runs as nonroot — /root is not writable. - // Use /tmp so subfinder can create its config dir. - HOME: '/tmp', + HOME: '/root', }, - command: [], + // Shell wrapper pattern: sh -c 'subfinder "$@"' -- [args...] + // This allows dynamic args to be appended and properly passed to subfinder + command: ['-c', 'subfinder "$@"', '--'], }, inputs: inputSchema, outputs: outputSchema, @@ -308,11 +303,10 @@ const definition = defineComponent({ 'Enumerate subdomains for a single target domain prior to Amass or Naabu.', 'Quick passive discovery during scope triage workflows.', ], - }, - toolProvider: { - kind: 'component', - name: 'subdomain_discovery', - description: 'Passive subdomain enumeration tool (Subfinder).', + agentTool: { + enabled: true, + toolDescription: 'Passive subdomain enumeration tool (Subfinder).', + }, }, async execute({ inputs, params }, context) { const parsedParams = parameterSchema.parse(params); @@ -366,7 +360,6 @@ const definition = defineComponent({ context.logger.info('[Subfinder] Skipping execution because no domains were provided.'); return { subdomains: [], - results: [], rawOutput: '', domainCount: 0, subdomainCount: 0, @@ -384,7 +377,7 @@ const definition = defineComponent({ const tenantId = (context as any).tenantId ?? 'default-tenant'; // Create isolated volume for this execution - const volume = new IsolatedContainerVolume(tenantId, context.runId); + const volume = createIsolatedVolume(tenantId, context.runId); const baseRunner = definition.runner; if (baseRunner.kind !== 'docker') { @@ -431,7 +424,9 @@ const definition = defineComponent({ network: baseRunner.network, timeoutSeconds: baseRunner.timeoutSeconds ?? SUBFINDER_TIMEOUT_SECONDS, env: { ...(baseRunner.env ?? {}) }, - // Pass subfinder CLI args directly (image default entrypoint is subfinder) + // Preserve the shell wrapper from baseRunner (sh -c 'subfinder "$@"' --) + entrypoint: baseRunner.entrypoint, + // Append subfinder arguments to shell wrapper command command: [...(baseRunner.command ?? []), ...subfinderArgs], volumes: [volume.getVolumeConfig(CONTAINER_INPUT_DIR, true)], }; @@ -516,22 +511,11 @@ const definition = defineComponent({ }); } - // Build analytics-ready results with scanner metadata - const analyticsResults: AnalyticsResult[] = subdomains.map((subdomain) => ({ - scanner: 'subfinder', - finding_hash: generateFindingHash('subdomain-discovery', subdomain, domains.join(',')), - severity: 'info' as const, - asset_key: subdomain, - subdomain, - parent_domains: domains, - })); - return { subdomains, rawOutput, domainCount, subdomainCount, - results: analyticsResults, }; }, }); diff --git a/worker/src/components/security/supabase-scanner.ts b/worker/src/components/security/supabase-scanner.ts index 5b1d3c96d..1a0968800 100644 --- a/worker/src/components/security/supabase-scanner.ts +++ b/worker/src/components/security/supabase-scanner.ts @@ -10,12 +10,9 @@ import { parameters, port, param, - generateFindingHash, - analyticsResultSchema, - type AnalyticsResult, - type DockerRunnerConfig, } from '@shipsec/component-sdk'; -import { IsolatedContainerVolume } from '../../utils/isolated-volume'; +import type { DockerRunnerConfig } from '@shipsec/component-sdk'; +import { createIsolatedVolume } from '../../utils/isolated-volume'; // Extract Supabase project ref from a standard URL like https://.supabase.co function inferProjectRef(supabaseUrl: string): string | null { @@ -153,11 +150,6 @@ const outputSchema = outputs({ reason: 'Scanner issue payloads can vary by Supabase project configuration.', connectionType: { kind: 'list', element: { kind: 'primitive', name: 'json' } }, }), - results: port(z.array(analyticsResultSchema()), { - label: 'Results', - description: - 'Analytics-ready findings with scanner, finding_hash, and severity. Connect to Analytics Sink.', - }), report: port(z.unknown(), { label: 'Scanner Report', description: 'Full JSON report produced by the scanner.', @@ -255,7 +247,7 @@ const definition = defineComponent({ }; const tenantId = (context as any).tenantId ?? 'default-tenant'; - const volume = new IsolatedContainerVolume(tenantId, context.runId); + const volume = createIsolatedVolume(tenantId, context.runId); const mountPath = '/data'; const configFilename = 'scanner_config.yaml'; const outputFilename = 'report.json'; @@ -337,19 +329,6 @@ const definition = defineComponent({ } catch (err) { const msg = (err as Error)?.message ?? 'Unknown error'; context.logger.error(`[SupabaseScanner] Scanner failed: ${msg}`); - - // Check if this is a fatal Docker error (image pull failure, container start failure) - // These should fail hard, not gracefully degrade - if ( - msg.includes('exit code 125') || - msg.includes('Unable to find image') || - msg.includes('permission denied') || - msg.includes('authentication required') - ) { - throw err; - } - - // For other errors (scanner runtime errors), allow graceful degradation errors.push(msg); } @@ -378,22 +357,6 @@ const definition = defineComponent({ } catch (err) { const msg = (err as Error)?.message ?? 'Unknown error'; context.logger.error(`[SupabaseScanner] Scanner failed: ${msg}`); - - // Check if this is a fatal Docker error that should fail the workflow - if ( - msg.includes('exit code 125') || - msg.includes('Unable to find image') || - msg.includes('permission denied') || - msg.includes('authentication required') - ) { - // Cleanup volume before throwing - if (volumeInitialized) { - await volume.cleanup(); - context.logger.info('[SupabaseScanner] Cleaned up isolated volume'); - } - throw err; - } - errors.push(msg); } finally { if (volumeInitialized) { @@ -402,34 +365,11 @@ const definition = defineComponent({ } } - // Build analytics-ready results with scanner metadata (follows core.analytics.result.v1 contract) - const results: AnalyticsResult[] = (issues ?? []).map((issue) => { - const issueObj = typeof issue === 'object' && issue !== null ? issue : { raw: issue }; - const issueRecord = issueObj as Record; - // Extract check_id and resource for deduplication hash - const checkId = issueRecord.check_id as string | undefined; - const resource = issueRecord.resource as string | undefined; - // Map severity from scanner output or default to 'medium' for security issues - const rawSeverity = (issueRecord.severity as string | undefined)?.toLowerCase(); - const validSeverities = ['critical', 'high', 'medium', 'low', 'info', 'none'] as const; - const severity = validSeverities.includes(rawSeverity as (typeof validSeverities)[number]) - ? (rawSeverity as (typeof validSeverities)[number]) - : 'medium'; - return { - ...issueObj, - scanner: 'supabase-scanner', - severity, - asset_key: projectRef ?? undefined, - finding_hash: generateFindingHash(checkId, projectRef, resource), - }; - }); - const output: Output = { projectRef: projectRef ?? null, score, summary, issues, - results, report, rawOutput: stdoutCombined ?? '', errors: errors.length > 0 ? errors : undefined, diff --git a/worker/src/components/security/trufflehog.ts b/worker/src/components/security/trufflehog.ts index 06a768bbe..72f4c19a6 100644 --- a/worker/src/components/security/trufflehog.ts +++ b/worker/src/components/security/trufflehog.ts @@ -12,11 +12,8 @@ import { parameters, port, param, - generateFindingHash, - analyticsResultSchema, - type AnalyticsResult, } from '@shipsec/component-sdk'; -import { IsolatedContainerVolume } from '../../utils/isolated-volume'; +import { createIsolatedVolume } from '../../utils/isolated-volume'; const scanTypeSchema = z.enum(['git', 'github', 'gitlab', 's3', 'gcs', 'filesystem', 'docker']); @@ -189,11 +186,6 @@ const outputSchema = outputs({ label: 'Has Verified Secrets', description: 'True when any verified secrets are detected.', }), - results: port(z.array(analyticsResultSchema()), { - label: 'Results', - description: - 'Analytics-ready findings with scanner, finding_hash, and severity. Connect to Analytics Sink.', - }), }); // Helper function to build TruffleHog command arguments @@ -264,7 +256,6 @@ function parseRawOutput(rawOutput: string): Output { secretCount: 0, verifiedCount: 0, hasVerifiedSecrets: false, - results: [], }; } @@ -303,7 +294,6 @@ function parseRawOutput(rawOutput: string): Output { secretCount: secrets.length, verifiedCount, hasVerifiedSecrets: verifiedCount > 0, - results: [], // Populated in execute() with scanner metadata }; } @@ -313,7 +303,7 @@ const definition = defineComponent({ category: 'security', runner: { kind: 'docker', - image: 'ghcr.io/shipsecai/trufflehog:latest', + image: 'ghcr.io/shipsecai/trufflehog:v3.93.1', entrypoint: 'trufflehog', network: 'bridge', command: [], // Will be built dynamically in execute @@ -333,11 +323,6 @@ const definition = defineComponent({ outputs: outputSchema, parameters: parameterSchema, docs: 'Scan for secrets and credentials using TruffleHog. Supports Git repositories, GitHub, GitLab, filesystems, S3 buckets, Docker images, and more.', - toolProvider: { - kind: 'component', - name: 'secret_scan', - description: 'Secret and credential leakage scanner (TruffleHog).', - }, ui: { slug: 'trufflehog', version: '1.0.0', @@ -364,6 +349,10 @@ const definition = defineComponent({ 'Scan only changes in a Pull Request by setting branch to PR branch and sinceCommit to base branch.', 'Scan last 10 commits in CI/CD using sinceCommit=HEAD~10 to catch recent secrets.', ], + agentTool: { + enabled: true, + toolDescription: 'Secret and credential leakage scanner (TruffleHog).', + }, }, async execute({ inputs, params }, context) { const parsedParams = parameterSchema.parse(params); @@ -392,7 +381,7 @@ const definition = defineComponent({ }); // Handle filesystem scanning with isolated volumes - let volume: IsolatedContainerVolume | undefined; + let volume: ReturnType | undefined; let effectiveInput = runnerPayload; const baseRunner = definition.runner; @@ -415,7 +404,7 @@ const definition = defineComponent({ } const tenantId = (context as any).tenantId ?? 'default-tenant'; - volume = new IsolatedContainerVolume(tenantId, context.runId); + volume = createIsolatedVolume(tenantId, context.runId); // Initialize volume with files const volumeName = await volume.initialize(runnerPayload.filesystemContent); @@ -504,23 +493,7 @@ const definition = defineComponent({ }); } - // Build analytics-ready results with scanner metadata (follows core.analytics.result.v1 contract) - const results: AnalyticsResult[] = output.secrets.map((secret: Secret) => { - // Extract file path from source metadata for hashing - const filePath = - secret.SourceMetadata?.Data?.Git?.file ?? - secret.SourceMetadata?.Data?.Filesystem?.file ?? - ''; - return { - ...secret, - scanner: 'trufflehog', - severity: 'high' as const, // Secrets are always high severity - asset_key: runnerPayload.scanTarget, - finding_hash: generateFindingHash(secret.DetectorType, secret.Redacted, filePath), - }; - }); - - return { ...output, results }; + return output; } finally { // Always cleanup volume if it was created if (volume) { diff --git a/worker/src/components/test/simple-http-mcp.ts b/worker/src/components/test/simple-http-mcp.ts index e8a72df6c..84cab7bd1 100644 --- a/worker/src/components/test/simple-http-mcp.ts +++ b/worker/src/components/test/simple-http-mcp.ts @@ -13,7 +13,7 @@ import { runComponentWithRunner, } from '@shipsec/component-sdk'; import { z } from 'zod'; -import { IsolatedContainerVolume } from '../../utils/isolated-volume'; +import { createIsolatedVolume } from '../../utils/isolated-volume'; const inputSchema = inputs({}); @@ -70,7 +70,7 @@ const definition = defineComponent({ async execute({ inputs: _inputs, params }, context) { const { port } = params; const { tenantId = 'default' } = context as any; - const volume = new IsolatedContainerVolume(tenantId, context.runId); + const volume = createIsolatedVolume(tenantId, context.runId); try { // Create MCP server script diff --git a/worker/src/temporal/activities/mcp-discovery.activity.ts b/worker/src/temporal/activities/mcp-discovery.activity.ts index cb0aa1bf0..498339e15 100644 --- a/worker/src/temporal/activities/mcp-discovery.activity.ts +++ b/worker/src/temporal/activities/mcp-discovery.activity.ts @@ -464,6 +464,8 @@ async function cleanupContainer(containerId: string | undefined): Promise if (!containerId) { return; } + // In K8s mode there is no Docker daemon — skip container cleanup + if (process.env.EXECUTION_MODE === 'k8s') return; // Validate container ID to prevent command injection if (!/^[a-zA-Z0-9_.-][a-zA-Z0-9_.-]*$/.test(containerId)) { console.warn(`[MCP Discovery] Skipping cleanup with unsafe container id: ${containerId}`); diff --git a/worker/src/temporal/activities/mcp.activity.ts b/worker/src/temporal/activities/mcp.activity.ts index ca2281417..d042a4777 100644 --- a/worker/src/temporal/activities/mcp.activity.ts +++ b/worker/src/temporal/activities/mcp.activity.ts @@ -2,12 +2,11 @@ import { componentRegistry, ConfigurationError, getCredentialInputIds, - isAgentCallable, getToolMetadata, ServiceError, } from '@shipsec/component-sdk'; import { - CleanupRunResourcesActivityInput, + CleanupLocalMcpActivityInput, RegisterComponentToolActivityInput, RegisterLocalMcpActivityInput, RegisterRemoteMcpActivityInput, @@ -69,112 +68,56 @@ export async function registerComponentToolActivity( export async function registerRemoteMcpActivity( input: RegisterRemoteMcpActivityInput, ): Promise { - await callInternalApi('register-mcp-server', { - runId: input.runId, - nodeId: input.nodeId, - serverName: input.toolName, - transport: 'http' as const, - endpoint: input.endpoint, - ...(input.authToken ? { headers: { Authorization: `Bearer ${input.authToken}` } } : {}), - }); + await callInternalApi('register-remote', input); } export async function registerLocalMcpActivity( input: RegisterLocalMcpActivityInput, ): Promise { const port = input.port || 8080; + // Use provided endpoint/containerId or fall back to defaults const endpoint = input.endpoint || `http://localhost:${port}`; const containerId = input.containerId || `docker-${input.image.replace(/[^a-zA-Z0-9]/g, '-')}`; - await callInternalApi('register-mcp-server', { - runId: input.runId, - nodeId: input.nodeId, - serverName: input.toolName, - transport: 'stdio' as const, + await callInternalApi('register-local', { + ...input, endpoint, containerId, }); } -// DEBUG: To disable container cleanup for inspecting Docker logs: -// Set environment variable: SKIP_CONTAINER_CLEANUP=true -// Or uncomment the line below: -// const SKIP_CLEANUP = true; -const SKIP_CONTAINER_CLEANUP = process.env.SKIP_CONTAINER_CLEANUP === 'true'; - -export async function cleanupRunResourcesActivity( - input: CleanupRunResourcesActivityInput, -): Promise { - // DEBUG: Skip cleanup to inspect Docker logs - if (SKIP_CONTAINER_CLEANUP) { - console.log( - `[MCP Cleanup] SKIP: Container cleanup disabled via SKIP_CONTAINER_CLEANUP env var`, - ); - console.log( - `[MCP Cleanup] Run 'docker ps -a | grep mcp' to see containers for run ${input.runId}`, - ); - return; - } - - const { exec } = await import('node:child_process'); - const { promisify } = await import('node:util'); - const execAsync = promisify(exec); +export async function cleanupLocalMcpActivity(input: CleanupLocalMcpActivityInput): Promise { + // In K8s mode there are no local Docker containers to clean up + if (process.env.EXECUTION_MODE === 'k8s') return; - // Get container IDs from tool registry (primary method) const response = (await callInternalApi('cleanup', { runId: input.runId })) as { containerIds?: string[]; }; - const registryContainerIds = Array.isArray(response?.containerIds) ? response.containerIds : []; + const containerIds = Array.isArray(response?.containerIds) ? response.containerIds : []; - // Fallback: Find containers by name pattern (catches orphaned containers) - // MCP containers follow the pattern: mcp-server-{image}-{timestamp} - let namePatternContainerIds: string[] = []; - try { - const { stdout } = await execAsync( - `docker ps -a --filter "name=mcp-server-" --format "{{.Names}}"`, - ); - namePatternContainerIds = stdout - .split('\n') - .map((line) => line.trim()) - .filter(Boolean); - - console.log( - `[MCP Cleanup] Found ${namePatternContainerIds.length} containers matching name pattern`, - ); - } catch (error) { - console.warn(`[MCP Cleanup] Failed to list containers by name pattern:`, error); + if (containerIds.length === 0) { + return; } - // Combine both sources and deduplicate - const allContainerIds = Array.from( - new Set([...registryContainerIds, ...namePatternContainerIds]), - ); + const { exec } = await import('node:child_process'); + const { promisify } = await import('node:util'); + const execAsync = promisify(exec); - console.log( - `[MCP Cleanup] Cleaning up ${allContainerIds.length} containers for run ${input.runId} ` + - `(${registryContainerIds.length} from registry, ${namePatternContainerIds.length} from name pattern)`, + await Promise.all( + containerIds.map(async (containerId: string) => { + if (!containerId || typeof containerId !== 'string') return; + if (!/^[a-zA-Z0-9_.-]+$/.test(containerId)) { + console.warn(`[MCP Cleanup] Skipping container with unsafe id: ${containerId}`); + return; + } + try { + await execAsync(`docker rm -f ${containerId}`); + } catch (error) { + console.warn(`[MCP Cleanup] Failed to remove container ${containerId}:`, error); + } + }), ); - if (allContainerIds.length === 0) { - console.log(`[MCP Cleanup] No containers to clean up for run ${input.runId}`); - } else { - await Promise.all( - allContainerIds.map(async (containerId: string) => { - if (!containerId || typeof containerId !== 'string') return; - if (!/^[a-zA-Z0-9_.-]+$/.test(containerId)) { - console.warn(`[MCP Cleanup] Skipping container with unsafe id: ${containerId}`); - return; - } - try { - await execAsync(`docker rm -f ${containerId}`); - console.log(`[MCP Cleanup] Removed container: ${containerId}`); - } catch (error) { - console.warn(`[MCP Cleanup] Failed to remove container ${containerId}:`, error); - } - }), - ); - } - if (!/^[a-zA-Z0-9_.-]+$/.test(input.runId)) { console.warn(`[MCP Cleanup] Skipping volume cleanup with unsafe runId: ${input.runId}`); return; @@ -236,7 +179,6 @@ export async function prepareAndRegisterToolActivity(input: { const metadata = getToolMetadata(component); const credentialIds = getCredentialInputIds(component); - const exposedToAgent = isAgentCallable(component); // Extract credentials from inputs/params const allInputs = { ...input.inputs, ...input.params }; @@ -250,12 +192,10 @@ export async function prepareAndRegisterToolActivity(input: { await callInternalApi('register-component', { runId: input.runId, nodeId: input.nodeId, - toolName: metadata.name || input.nodeId.replace(/[^a-zA-Z0-9]/g, '_'), - exposedToAgent, + toolName: input.nodeId.replace(/[^a-zA-Z0-9]/g, '_'), componentId: input.componentId, description: metadata.description, inputSchema: metadata.inputSchema, - parameters: input.params, credentials, }); } From 0be791374c91a1ad003afbace31b6bb8a556a6ec Mon Sep 17 00:00:00 2001 From: betterclever Date: Thu, 12 Feb 2026 02:39:05 +0400 Subject: [PATCH 005/690] feat(frontend): allow custom hosted studio domain --- frontend/vite.config.ts | 48 +++-------------------------------------- 1 file changed, 3 insertions(+), 45 deletions(-) diff --git a/frontend/vite.config.ts b/frontend/vite.config.ts index 98ba0807f..20b906167 100644 --- a/frontend/vite.config.ts +++ b/frontend/vite.config.ts @@ -2,10 +2,6 @@ import { defineConfig } from 'vite'; import react from '@vitejs/plugin-react'; import path from 'path'; -const instance = parseInt(process.env.SHIPSEC_INSTANCE || '0', 10); -const frontendPort = 5173 + instance * 100; -const backendPort = 3211 + instance * 100; - // https://vitejs.dev/config/ export default defineConfig({ plugins: [react()], @@ -25,49 +21,11 @@ export default defineConfig({ }, server: { host: '0.0.0.0', - port: frontendPort, - strictPort: true, + port: 5173, open: false, - allowedHosts: ['studio.shipsec.ai', 'frontend'], - proxy: { - '/api/': { - target: `http://localhost:${backendPort}`, - changeOrigin: true, - secure: false, - }, - '/analytics/': { - target: 'http://localhost:5601', - changeOrigin: true, - secure: false, - }, - }, + allowedHosts: ['studio.shipsec.ai', 'studio-next.shipsec.ai'], }, preview: { - allowedHosts: ['studio.shipsec.ai', 'frontend'], - }, - build: { - rollupOptions: { - output: { - manualChunks: { - 'vendor-react': ['react', 'react-dom', 'react-router-dom'], - 'vendor-radix': [ - '@radix-ui/react-accordion', - '@radix-ui/react-avatar', - '@radix-ui/react-checkbox', - '@radix-ui/react-dialog', - '@radix-ui/react-dropdown-menu', - '@radix-ui/react-label', - '@radix-ui/react-popover', - '@radix-ui/react-select', - '@radix-ui/react-slider', - '@radix-ui/react-slot', - '@radix-ui/react-switch', - '@radix-ui/react-tabs', - '@radix-ui/react-tooltip', - ], - 'vendor-analytics': ['posthog-js'], - }, - }, - }, + allowedHosts: ['studio.shipsec.ai', 'studio-next.shipsec.ai'], }, }); From b582b597985ccedc45b9b69c210234fb39a297e6 Mon Sep 17 00:00:00 2001 From: betterclever Date: Thu, 12 Feb 2026 03:26:02 +0400 Subject: [PATCH 006/690] fix(frontend): use relative API URL for path-based routing --- frontend/src/services/api.ts | 187 ++--------------------------------- 1 file changed, 9 insertions(+), 178 deletions(-) diff --git a/frontend/src/services/api.ts b/frontend/src/services/api.ts index 727f41c31..1ad9169e4 100644 --- a/frontend/src/services/api.ts +++ b/frontend/src/services/api.ts @@ -35,9 +35,8 @@ type ApiKeyResponseDto = components['schemas']['ApiKeyResponseDto']; type CreateApiKeyResponseDto = components['schemas']['CreateApiKeyResponseDto']; type CreateApiKeyDto = components['schemas']['CreateApiKeyDto']; type UpdateApiKeyDto = components['schemas']['UpdateApiKeyDto']; -type ListAuditLogsResponseDto = components['schemas']['ListAuditLogsResponseDto']; -export interface TerminalChunkResponse { +interface TerminalChunkResponse { runId: string; cursor?: string; chunks: { @@ -52,21 +51,6 @@ export interface TerminalChunkResponse { }[]; } -export interface WorkflowSummary { - id: string; - name: string; - description: string | null; - organizationId: string | null; - isSystem: boolean; - templateId: string | null; - lastRun: string | null; - latestRunStatus: string | null; - runCount: number; - nodeCount: number; - createdAt: string; - updatedAt: string; -} - export type IntegrationProvider = IntegrationProviderResponse; export type IntegrationConnection = IntegrationConnectionResponse; export type IntegrationProviderConfiguration = ProviderConfigurationResponse; @@ -99,6 +83,13 @@ function resolveApiBaseUrl() { } } + // When no explicit API URL is configured, use same-origin relative path. + // Works with path-based routing (e.g. /api/v1/* routed to backend via Ingress). + // Falls back to localhost only in local dev where Vite proxy handles it. + if (typeof window !== 'undefined' && window.location.hostname !== 'localhost') { + return ''; + } + return 'http://localhost:3211'; } @@ -189,122 +180,6 @@ async function fetchScheduleById(id: string): Promise { * Simple wrapper around the backend API client */ export const api = { - templates: { - list: async (params?: { category?: string; search?: string; tags?: string[] }) => { - const searchParams = new URLSearchParams(); - if (params?.category) searchParams.set('category', params.category); - if (params?.search) searchParams.set('search', params.search); - if (params?.tags) searchParams.set('tags', params.tags.join(',')); - - const headers = await getAuthHeaders(); - const response = await fetch( - `${API_V1_URL}/templates${searchParams.toString() ? `?${searchParams.toString()}` : ''}`, - { headers }, - ); - - if (!response.ok) throw new Error('Failed to fetch templates'); - return response.json(); - }, - - get: async (id: string) => { - const headers = await getAuthHeaders(); - const response = await fetch(`${API_V1_URL}/templates/${id}`, { headers }); - if (!response.ok) throw new Error('Failed to fetch template'); - return response.json(); - }, - - getCategories: async () => { - const headers = await getAuthHeaders(); - const response = await fetch(`${API_V1_URL}/templates/categories`, { headers }); - if (!response.ok) throw new Error('Failed to fetch categories'); - return response.json(); - }, - - getTags: async () => { - const headers = await getAuthHeaders(); - const response = await fetch(`${API_V1_URL}/templates/tags`, { headers }); - if (!response.ok) throw new Error('Failed to fetch tags'); - return response.json(); - }, - - publish: async (data: { - workflowId: string; - name: string; - description: string; - category: string; - tags: string[]; - author: string; - }) => { - const headers = await getAuthHeaders(); - const response = await fetch(`${API_V1_URL}/templates/publish`, { - method: 'POST', - headers: { - ...headers, - 'Content-Type': 'application/json', - }, - body: JSON.stringify(data), - }); - - if (!response.ok) { - const errorData = await response - .json() - .catch(() => ({ message: 'Failed to publish template' })); - throw new Error(errorData.message || 'Failed to publish template'); - } - - return response.json(); - }, - - use: async ( - templateId: string, - data: { workflowName: string; secretMappings?: Record }, - ) => { - const headers = await getAuthHeaders(); - const response = await fetch(`${API_V1_URL}/templates/${templateId}/use`, { - method: 'POST', - headers: { - ...headers, - 'Content-Type': 'application/json', - }, - body: JSON.stringify(data), - }); - - if (!response.ok) { - const errorData = await response - .json() - .catch(() => ({ message: 'Failed to use template' })); - throw new Error(errorData.message || 'Failed to use template'); - } - - return response.json(); - }, - - sync: async () => { - const headers = await getAuthHeaders(); - const response = await fetch(`${API_V1_URL}/templates/sync`, { - method: 'POST', - headers, - }); - - if (!response.ok) throw new Error('Failed to sync templates'); - return response.json(); - }, - - getMySubmissions: async () => { - const headers = await getAuthHeaders(); - const response = await fetch(`${API_V1_URL}/templates/my`, { headers }); - if (!response.ok) throw new Error('Failed to fetch submissions'); - return response.json(); - }, - - getSubmissions: async () => { - const headers = await getAuthHeaders(); - const response = await fetch(`${API_V1_URL}/templates/submissions`, { headers }); - if (!response.ok) throw new Error('Failed to fetch submissions'); - return response.json(); - }, - }, - workflows: { list: async (): Promise => { const response = await apiClient.listWorkflows(); @@ -312,13 +187,6 @@ export const api = { return response.data || []; }, - listSummary: async (): Promise => { - const headers = await getAuthHeaders(); - const response = await fetch(`${API_V1_URL}/workflows/summary`, { headers }); - if (!response.ok) throw new Error('Failed to fetch workflow summaries'); - return response.json(); - }, - get: async (id: string): Promise => { const response = await apiClient.getWorkflow(id); if (response.error) throw new Error('Failed to fetch workflow'); @@ -645,38 +513,6 @@ export const api = { }, }, - auditLogs: { - list: async (query: { - resourceType?: string; - resourceId?: string; - action?: string; - actorId?: string; - from?: string; - to?: string; - limit?: number; - cursor?: string; - }): Promise => { - const headers = await getAuthHeaders(); - const url = new URL(`${API_V1_URL}/audit-logs`); - - if (query.resourceType) url.searchParams.set('resourceType', query.resourceType); - if (query.resourceId) url.searchParams.set('resourceId', query.resourceId); - if (query.action) url.searchParams.set('action', query.action); - if (query.actorId) url.searchParams.set('actorId', query.actorId); - if (query.from) url.searchParams.set('from', query.from); - if (query.to) url.searchParams.set('to', query.to); - if (query.cursor) url.searchParams.set('cursor', query.cursor); - if (query.limit) url.searchParams.set('limit', String(query.limit)); - - const res = await fetch(url.toString(), { headers }); - if (!res.ok) { - const text = await res.text(); - throw new Error(`Failed to fetch audit logs: ${res.status} ${text}`); - } - return (await res.json()) as ListAuditLogsResponseDto; - }, - }, - executions: { start: async ( workflowId: string, @@ -850,12 +686,7 @@ export const api = { return { success: true }; }, - listRuns: async (options?: { - workflowId?: string; - status?: string; - limit?: number; - offset?: number; - }) => { + listRuns: async (options?: { workflowId?: string; status?: string; limit?: number }) => { const response = await apiClient.listWorkflowRuns(options); if (response.error) throw new Error('Failed to fetch runs'); return response.data || { runs: [] }; From 58b5fcd6db2642fbc01253ee367ff68268838774 Mon Sep 17 00:00:00 2001 From: betterclever Date: Thu, 12 Feb 2026 03:59:18 +0400 Subject: [PATCH 007/690] fix(frontend): remove all hardcoded localhost:3211 defaults --- Dockerfile | 12 ++++-------- frontend/src/services/api.ts | 12 ++++-------- packages/backend-client/src/api-client.ts | 4 +--- 3 files changed, 9 insertions(+), 19 deletions(-) diff --git a/Dockerfile b/Dockerfile index c2f28972e..283cb442a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -83,13 +83,12 @@ FROM base AS frontend # Frontend build-time configuration ARG VITE_AUTH_PROVIDER=local ARG VITE_CLERK_PUBLISHABLE_KEY="" -ARG VITE_API_URL=http://localhost:3211 -ARG VITE_BACKEND_URL=http://localhost:3211 +ARG VITE_API_URL="" +ARG VITE_BACKEND_URL="" ARG VITE_DEFAULT_ORG_ID=local-dev ARG VITE_GIT_SHA=unknown ARG VITE_PUBLIC_POSTHOG_KEY="" ARG VITE_PUBLIC_POSTHOG_HOST="" -ARG VITE_OPENSEARCH_DASHBOARDS_URL="" ENV VITE_AUTH_PROVIDER=${VITE_AUTH_PROVIDER} ENV VITE_CLERK_PUBLISHABLE_KEY=${VITE_CLERK_PUBLISHABLE_KEY} @@ -99,7 +98,6 @@ ENV VITE_DEFAULT_ORG_ID=${VITE_DEFAULT_ORG_ID} ENV VITE_GIT_SHA=${VITE_GIT_SHA} ENV VITE_PUBLIC_POSTHOG_KEY=${VITE_PUBLIC_POSTHOG_KEY} ENV VITE_PUBLIC_POSTHOG_HOST=${VITE_PUBLIC_POSTHOG_HOST} -ENV VITE_OPENSEARCH_DASHBOARDS_URL=${VITE_OPENSEARCH_DASHBOARDS_URL} # Set working directory for frontend USER shipsec @@ -125,13 +123,12 @@ FROM base AS frontend-debug # Frontend build-time configuration ARG VITE_AUTH_PROVIDER=local ARG VITE_CLERK_PUBLISHABLE_KEY="" -ARG VITE_API_URL=http://localhost:3211 -ARG VITE_BACKEND_URL=http://localhost:3211 +ARG VITE_API_URL="" +ARG VITE_BACKEND_URL="" ARG VITE_DEFAULT_ORG_ID=local-dev ARG VITE_GIT_SHA=unknown ARG VITE_PUBLIC_POSTHOG_KEY="" ARG VITE_PUBLIC_POSTHOG_HOST="" -ARG VITE_OPENSEARCH_DASHBOARDS_URL="" ENV VITE_AUTH_PROVIDER=${VITE_AUTH_PROVIDER} ENV VITE_CLERK_PUBLISHABLE_KEY=${VITE_CLERK_PUBLISHABLE_KEY} @@ -141,7 +138,6 @@ ENV VITE_DEFAULT_ORG_ID=${VITE_DEFAULT_ORG_ID} ENV VITE_GIT_SHA=${VITE_GIT_SHA} ENV VITE_PUBLIC_POSTHOG_KEY=${VITE_PUBLIC_POSTHOG_KEY} ENV VITE_PUBLIC_POSTHOG_HOST=${VITE_PUBLIC_POSTHOG_HOST} -ENV VITE_OPENSEARCH_DASHBOARDS_URL=${VITE_OPENSEARCH_DASHBOARDS_URL} # Set working directory for frontend USER shipsec diff --git a/frontend/src/services/api.ts b/frontend/src/services/api.ts index 1ad9169e4..d82d2b195 100644 --- a/frontend/src/services/api.ts +++ b/frontend/src/services/api.ts @@ -83,14 +83,10 @@ function resolveApiBaseUrl() { } } - // When no explicit API URL is configured, use same-origin relative path. - // Works with path-based routing (e.g. /api/v1/* routed to backend via Ingress). - // Falls back to localhost only in local dev where Vite proxy handles it. - if (typeof window !== 'undefined' && window.location.hostname !== 'localhost') { - return ''; - } - - return 'http://localhost:3211'; + // No explicit API URL — use same-origin relative paths. + // Works with path-based routing (/api/v1/* routed to backend via Ingress). + // For local dev, set VITE_API_URL=http://localhost:3211 in frontend/.env + return ''; } export const API_BASE_URL = resolveApiBaseUrl(); diff --git a/packages/backend-client/src/api-client.ts b/packages/backend-client/src/api-client.ts index 4378d1ed8..1139dd0b2 100644 --- a/packages/backend-client/src/api-client.ts +++ b/packages/backend-client/src/api-client.ts @@ -36,7 +36,7 @@ export class ShipSecApiClient { private baseUrl: string; constructor(config: ClientConfig = {}) { - this.baseUrl = config.baseUrl || 'http://localhost:3211'; + this.baseUrl = config.baseUrl || ''; this.client = createClient({ baseUrl: this.baseUrl, @@ -187,7 +187,6 @@ export class ShipSecApiClient { workflowId?: string; status?: string; limit?: number; - offset?: number; }) { return this.client.GET('/api/v1/workflows/runs', { params: { @@ -195,7 +194,6 @@ export class ShipSecApiClient { workflowId: options?.workflowId, status: options?.status, limit: options?.limit, - offset: options?.offset, }, }, }); From 7e1f82f0c1236d317fca547f4ad2684bd33698ae Mon Sep 17 00:00:00 2001 From: betterclever Date: Thu, 12 Feb 2026 12:05:18 +0400 Subject: [PATCH 008/690] fix(frontend): handle relative URL in terminal chunks fetch --- frontend/src/services/api.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frontend/src/services/api.ts b/frontend/src/services/api.ts index d82d2b195..e9848ced7 100644 --- a/frontend/src/services/api.ts +++ b/frontend/src/services/api.ts @@ -586,7 +586,8 @@ export const api = { }, ): Promise => { const headers = await getAuthHeaders(); - const url = new URL(`${API_V1_URL}/workflows/runs/${executionId}/terminal`); + const path = `${API_V1_URL}/workflows/runs/${executionId}/terminal`; + const url = new URL(path, window.location.origin); if (params?.nodeRef) url.searchParams.set('nodeRef', params.nodeRef); if (params?.stream) url.searchParams.set('stream', params.stream); if (params?.cursor) url.searchParams.set('cursor', params.cursor); From e4c8eac7ff8441c6319f7e7caeb3b703d7d3a9e6 Mon Sep 17 00:00:00 2001 From: betterclever Date: Thu, 12 Feb 2026 13:10:02 +0400 Subject: [PATCH 009/690] fix(worker): wait for container running before streaming runtime logs --- worker/src/utils/k8s-runner.ts | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/worker/src/utils/k8s-runner.ts b/worker/src/utils/k8s-runner.ts index edf7747a7..2b8f1feb5 100644 --- a/worker/src/utils/k8s-runner.ts +++ b/worker/src/utils/k8s-runner.ts @@ -386,11 +386,31 @@ async function waitForJobCompletion( * Stream pod logs to the context logger and terminal collector. * Uses the K8s Log API with a writable stream to capture output in real-time. */ +async function waitForContainerRunning( + podName: string, + namespace: string, + timeoutMs = 60_000, +): Promise { + const core = getCoreApi(); + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + const pod = await core.readNamespacedPod({ name: podName, namespace }); + const containerStatus = pod.status?.containerStatuses?.find((c) => c.name === 'component'); + if (containerStatus?.state?.running || containerStatus?.state?.terminated) { + return; + } + await new Promise((r) => setTimeout(r, 1000)); + } +} + async function streamPodLogs( podName: string, namespace: string, context: ExecutionContext, ): Promise { + // Wait for container to be running before streaming logs + await waitForContainerRunning(podName, namespace); + const kc = getKubeConfig(); const log = new k8s.Log(kc); From f425fe002b072f2814229c99226232d3387edf6b Mon Sep 17 00:00:00 2001 From: betterclever Date: Thu, 12 Feb 2026 17:45:31 +0400 Subject: [PATCH 010/690] fix(worker): read terminated pod logs for terminal streaming --- worker/src/utils/k8s-runner.ts | 71 +++++++++++++++++++++------------- 1 file changed, 45 insertions(+), 26 deletions(-) diff --git a/worker/src/utils/k8s-runner.ts b/worker/src/utils/k8s-runner.ts index 2b8f1feb5..cb7c99cbe 100644 --- a/worker/src/utils/k8s-runner.ts +++ b/worker/src/utils/k8s-runner.ts @@ -386,40 +386,32 @@ async function waitForJobCompletion( * Stream pod logs to the context logger and terminal collector. * Uses the K8s Log API with a writable stream to capture output in real-time. */ -async function waitForContainerRunning( - podName: string, - namespace: string, - timeoutMs = 60_000, -): Promise { - const core = getCoreApi(); - const deadline = Date.now() + timeoutMs; - while (Date.now() < deadline) { - const pod = await core.readNamespacedPod({ name: podName, namespace }); - const containerStatus = pod.status?.containerStatuses?.find((c) => c.name === 'component'); - if (containerStatus?.state?.running || containerStatus?.state?.terminated) { - return; - } - await new Promise((r) => setTimeout(r, 1000)); - } -} - async function streamPodLogs( podName: string, namespace: string, context: ExecutionContext, ): Promise { - // Wait for container to be running before streaming logs - await waitForContainerRunning(podName, namespace); - + const core = getCoreApi(); const kc = getKubeConfig(); const log = new k8s.Log(kc); - const { PassThrough } = await import('stream'); - const logStream = new PassThrough(); + // Wait for container to be ready (running or already terminated) + const deadline = Date.now() + 60_000; + let containerTerminated = false; + while (Date.now() < deadline) { + const pod = await core.readNamespacedPod({ name: podName, namespace }); + const cs = pod.status?.containerStatuses?.find((c) => c.name === 'component'); + if (cs?.state?.terminated) { + containerTerminated = true; + break; + } + if (cs?.state?.running) { + break; + } + await new Promise((r) => setTimeout(r, 500)); + } - logStream.on('data', (chunk: Buffer) => { - const text = chunk.toString(); - // Feed to terminal collector for real-time UI streaming + const emitToCollectors = (text: string) => { if (context.terminalCollector) { context.terminalCollector({ runId: context.runId, @@ -432,7 +424,6 @@ async function streamPodLogs( origin: 'k8s-job', }); } - // Also feed to log collector if (context.logCollector) { context.logCollector({ runId: context.runId, @@ -443,6 +434,34 @@ async function streamPodLogs( timestamp: new Date().toISOString(), }); } + }; + + // If container already terminated, read final logs instead of following + if (containerTerminated) { + try { + const logResponse = await core.readNamespacedPodLog({ + name: podName, + namespace, + container: 'component', + }); + const logText = typeof logResponse === 'string' ? logResponse : String(logResponse); + if (logText) { + emitToCollectors(logText); + } + } catch (err) { + context.logger.warn( + `[K8sRunner] Failed to read terminated pod logs: ${(err as Error).message}`, + ); + } + return; + } + + // Container is running — stream logs in real-time + const { PassThrough } = await import('stream'); + const logStream = new PassThrough(); + + logStream.on('data', (chunk: Buffer) => { + emitToCollectors(chunk.toString()); }); try { From 1b9173ab6df463e50112948c4d4acb910428fad8 Mon Sep 17 00:00:00 2001 From: betterclever Date: Thu, 12 Feb 2026 18:17:36 +0400 Subject: [PATCH 011/690] fix(worker): emit terminal chunks as PTY with base64 encoding --- packages/component-sdk/src/types.ts | 111 ++++++++++------------------ worker/src/utils/k8s-runner.ts | 9 ++- 2 files changed, 44 insertions(+), 76 deletions(-) diff --git a/packages/component-sdk/src/types.ts b/packages/component-sdk/src/types.ts index a01bf5d7c..13055a514 100644 --- a/packages/component-sdk/src/types.ts +++ b/packages/component-sdk/src/types.ts @@ -1,5 +1,4 @@ import { z } from 'zod'; -import type { ComponentCategory } from '@shipsec/shared'; import type { IArtifactService, @@ -15,7 +14,7 @@ import type { HttpInstrumentationOptions, HttpRequestInput } from './http/types' export type { ExecutionContextMetadata } from './interfaces'; -export type RunnerKind = 'inline' | 'docker' | 'remote'; +export type RunnerKind = 'inline' | 'docker' | 'remote' | 'k8s'; export interface InlineRunnerConfig { kind: 'inline'; @@ -91,65 +90,6 @@ export interface LogEventInput { metadata?: ExecutionContextMetadata; } -export interface McpServerSpec { - id: string; - name: string; - command: string; - args?: string[]; -} - -export type ToolProviderKind = - | 'component' // Component exposes itself as a tool - | 'mcp-server' // Component runs a single MCP server - | 'mcp-group'; // Component manages multiple MCP servers - -export interface ToolProviderConfig { - kind: ToolProviderKind; - - /** - * Tool name exposed to the agent. - * For 'component' kind, this is the tool name. - * For 'mcp-group', this is used as a prefix for child tools if needed. - */ - name: string; - - /** - * Description of what the tool(s) do, shown to the agent. - */ - description: string; - - /** - * Configuration for MCP-based tool providers. - * Required for 'mcp-server' and 'mcp-group' kinds. - */ - mcp?: { - /** Docker image to use for the MCP server(s) */ - image?: string; - /** Command to run if image is used (for 'mcp-server') */ - command?: string[]; - /** Mapping of environment variables to component inputs/params */ - credentialMapping?: Record; - /** Specification for individual servers in a group (for 'mcp-group') */ - servers?: McpServerSpec[]; - }; - - /** - * For 'component' kind, optional override for tool input schema. - * If not provided, it's inferred from component inputs. - */ - inputSchema?: any; - - /** - * Optional Docker configuration for 'component' kind tools that run via Docker - * but aren't full MCP servers (e.g., standard scanners). - */ - docker?: { - image: string; - command: string[]; - args?: string[]; - }; -} - export interface AgentTracePart { type: string; [key: string]: unknown; @@ -331,8 +271,7 @@ export type ComponentParameterType = | 'artifact' | 'variable-list' | 'form-fields' - | 'selection-options' - | 'analytics-inputs'; + | 'selection-options'; export interface ComponentParameterOption { label: string; @@ -365,6 +304,18 @@ export interface ComponentAuthorMetadata { url?: string; } +// Categories supported by the new functional grouping plus legacy values for backwards compatibility +export type ComponentCategory = + | 'input' + | 'transform' + | 'ai' + | 'mcp' + | 'security' + | 'it_ops' + | 'notification' + | 'manual_action' + | 'output'; + export type ComponentUiType = | 'trigger' | 'input' @@ -372,6 +323,24 @@ export type ComponentUiType = | 'process' | 'output'; +/** + * Configuration for exposing a component as an agent-callable tool. + */ +export interface AgentToolConfig { + /** Whether this component can be used as an agent tool */ + enabled: boolean; + /** + * Tool name exposed to the agent. Defaults to component slug with underscores. + * Should be descriptive and follow snake_case convention. + * @example 'check_ip_reputation', 'query_cloudtrail' + */ + toolName?: string; + /** + * Description of what the tool does, shown to the agent. + * Should clearly explain the tool's purpose and when to use it. + */ + toolDescription?: string; +} export interface ComponentUiMetadata { slug: string; @@ -390,6 +359,12 @@ export interface ComponentUiMetadata { examples?: string[]; /** UI-only component - should not be included in workflow execution */ uiOnly?: boolean; + /** + * Configuration for exposing this component as an agent-callable tool. + * When enabled, the component can be used in tool mode within workflows, + * allowing AI agents to invoke it via the MCP gateway. + */ + agentTool?: AgentToolConfig; } export interface ExecutionContext { @@ -402,11 +377,6 @@ export interface ExecutionContext { metadata: ExecutionContextMetadata; agentTracePublisher?: AgentTracePublisher; - // Workflow context (optional, available when running in workflow) - workflowId?: string; - workflowName?: string; - organizationId?: string | null; - // Service interfaces - implemented by adapters storage?: IFileStorageService; secrets?: ISecretsService; @@ -516,11 +486,6 @@ export interface ComponentDefinition< ui?: ComponentUiMetadata; requiresSecrets?: boolean; - /** - * Configuration for exposing this component (or its children) as agent-callable tools. - */ - toolProvider?: ToolProviderConfig; - /** Retry policy for this component (optional, uses default if not specified) */ retryPolicy?: ComponentRetryPolicy; diff --git a/worker/src/utils/k8s-runner.ts b/worker/src/utils/k8s-runner.ts index cb7c99cbe..0a438c49d 100644 --- a/worker/src/utils/k8s-runner.ts +++ b/worker/src/utils/k8s-runner.ts @@ -411,17 +411,20 @@ async function streamPodLogs( await new Promise((r) => setTimeout(r, 500)); } + let chunkIndex = 0; const emitToCollectors = (text: string) => { if (context.terminalCollector) { + chunkIndex += 1; context.terminalCollector({ runId: context.runId, nodeRef: context.componentRef, - stream: 'stdout', - chunkIndex: 0, - payload: text, + stream: 'pty', + chunkIndex, + payload: Buffer.from(text).toString('base64'), recordedAt: new Date().toISOString(), deltaMs: 0, origin: 'k8s-job', + runnerKind: 'k8s', }); } if (context.logCollector) { From bcc48ce78ef266f66647316ad9e82b4b2ea137dd Mon Sep 17 00:00:00 2001 From: betterclever Date: Fri, 13 Feb 2026 11:09:13 +0400 Subject: [PATCH 012/690] feat(worker): enable TTY on runtime Job containers for ANSI terminal... --- worker/src/utils/k8s-runner.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/worker/src/utils/k8s-runner.ts b/worker/src/utils/k8s-runner.ts index 0a438c49d..c3461d533 100644 --- a/worker/src/utils/k8s-runner.ts +++ b/worker/src/utils/k8s-runner.ts @@ -304,6 +304,7 @@ function buildJobSpec( command: command.length > 0 ? command : undefined, args: args.length > 0 ? args : undefined, env: envVars, + tty: true, volumeMounts, resources: { requests: { cpu: '100m', memory: '128Mi' }, From cf14b83718400baf6de47c587cc2606529cff61b Mon Sep 17 00:00:00 2001 From: betterclever Date: Fri, 13 Feb 2026 11:53:13 +0400 Subject: [PATCH 013/690] feat(worker): track deltaMs timing in runtime terminal stream chunks --- worker/src/utils/k8s-runner.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/worker/src/utils/k8s-runner.ts b/worker/src/utils/k8s-runner.ts index c3461d533..4232f592f 100644 --- a/worker/src/utils/k8s-runner.ts +++ b/worker/src/utils/k8s-runner.ts @@ -413,17 +413,21 @@ async function streamPodLogs( } let chunkIndex = 0; + let lastTimestamp = Date.now(); const emitToCollectors = (text: string) => { if (context.terminalCollector) { chunkIndex += 1; + const now = Date.now(); + const deltaMs = chunkIndex === 1 ? 0 : Math.max(0, now - lastTimestamp); + lastTimestamp = now; context.terminalCollector({ runId: context.runId, nodeRef: context.componentRef, stream: 'pty', chunkIndex, payload: Buffer.from(text).toString('base64'), - recordedAt: new Date().toISOString(), - deltaMs: 0, + recordedAt: new Date(now).toISOString(), + deltaMs, origin: 'k8s-job', runnerKind: 'k8s', }); From e3206d532ad6622224c901776e854a45b2b63283 Mon Sep 17 00:00:00 2001 From: betterclever Date: Fri, 13 Feb 2026 15:03:06 +0400 Subject: [PATCH 014/690] feat(worker): use runtime Attach API for live PTY streaming --- worker/src/utils/k8s-runner.ts | 57 ++++++++++++++++++++++++++++------ 1 file changed, 47 insertions(+), 10 deletions(-) diff --git a/worker/src/utils/k8s-runner.ts b/worker/src/utils/k8s-runner.ts index 4232f592f..c91ba62e9 100644 --- a/worker/src/utils/k8s-runner.ts +++ b/worker/src/utils/k8s-runner.ts @@ -464,22 +464,59 @@ async function streamPodLogs( return; } - // Container is running — stream logs in real-time - const { PassThrough } = await import('stream'); - const logStream = new PassThrough(); + // Container is running — attach to PTY for real-time streaming + const { Writable } = await import('stream'); - logStream.on('data', (chunk: Buffer) => { - emitToCollectors(chunk.toString()); + const stdoutSink = new Writable({ + write(chunk: Buffer, _encoding, callback) { + emitToCollectors(chunk.toString()); + callback(); + }, }); try { - await log.log(namespace, podName, 'component', logStream, { - follow: true, - pretty: false, - timestamps: false, + context.logger.info(`[K8sRunner] Attaching to pod ${podName} with TTY for live PTY stream`); + const attach = new k8s.Attach(kc); + const ws = await attach.attach( + namespace, + podName, + 'component', + stdoutSink, + stdoutSink, + null, + true, + ); + + // Wait for the WebSocket to close (container exits → WS closes) + await new Promise((resolve, reject) => { + ws.onclose = () => { + context.logger.info(`[K8sRunner] Attach WebSocket closed for pod ${podName}`); + resolve(); + }; + ws.onerror = (event) => { + context.logger.warn(`[K8sRunner] Attach WebSocket error: ${String(event)}`); + reject(new Error('Attach WebSocket error')); + }; }); } catch (err) { - context.logger.warn(`[K8sRunner] Log streaming failed: ${(err as Error).message}`); + context.logger.warn( + `[K8sRunner] Attach failed, falling back to log stream: ${(err as Error).message}`, + ); + // Fallback to log API if attach fails + const { PassThrough } = await import('stream'); + const logStream = new PassThrough(); + logStream.on('data', (chunk: Buffer) => { + emitToCollectors(chunk.toString()); + }); + try { + await log.log(namespace, podName, 'component', logStream, { + follow: true, + pretty: false, + timestamps: false, + }); + } catch (logErr) { + context.logger.warn(`[K8sRunner] Log streaming also failed: ${(logErr as Error).message}`); + } } } From 348f8ca680be8e9663f71418e2ee74042ec41828 Mon Sep 17 00:00:00 2001 From: Aseem Shrey Date: Tue, 10 Feb 2026 20:52:36 -0500 Subject: [PATCH 015/690] feat(backend): add GitHub App integration with webhooks, scans, and check runs --- .gitignore | 21 +- backend/.env.docker | 14 +- backend/.env.example | 23 + .../0020_add-fail-on-trigger-rules.sql | 5 + .../0021_widen-github-ids-to-bigint.sql | 4 + backend/drizzle/0022_add-pr-review-fields.sql | 6 + backend/drizzle/meta/_journal.json | 18 +- backend/src/app.module.ts | 10 +- .../src/components/utils/categorization.ts | 120 +- backend/src/config/github-app.config.ts | 67 + backend/src/database/schema/github-app.ts | 252 +++ backend/src/database/schema/index.ts | 4 +- backend/src/dsl/validator.ts | 50 +- .../__tests__/diff-hunk-parser.spec.ts | 84 + .../__tests__/github-app.service.spec.ts | 222 +++ .../scan-result-sync.service.spec.ts | 1085 ++++++++++++ backend/src/github-app/diff-hunk-parser.ts | 81 + backend/src/github-app/dto/github-app.dto.ts | 259 +++ .../src/github-app/github-app.controller.ts | 398 +++++ backend/src/github-app/github-app.module.ts | 31 + .../src/github-app/github-app.repository.ts | 469 +++++ backend/src/github-app/github-app.service.ts | 1533 +++++++++++++++++ backend/src/github-app/index.ts | 4 + .../github-app/scan-result-sync.service.ts | 787 +++++++++ backend/src/main.ts | 1 + .../__tests__/workflows.service.spec.ts | 43 +- backend/src/workflows/workflows.module.ts | 2 + backend/src/workflows/workflows.service.ts | 472 ++--- docs/GITHUB-SECURITY-WORKFLOWS.md | 137 ++ docs/samples/01-opengrep-repo-scan.json | 134 ++ docs/samples/02-trufflehog-secret-scan.json | 98 ++ .../03-security-scan-with-pr-comments.json | 263 +++ docs/samples/04-pr-security-scan.json | 436 +++++ docs/samples/05-pr-security-check-run.json | 418 +++++ docs/samples/06-opengrep-pr-repo-scan.json | 229 +++ docs/samples/07-ai-code-review.json | 200 +++ docs/samples/07-opengrep-pr-trigger.json | 13 + .../08-unified-ai-security-review.json | 282 +++ openapi.json | 1227 ++++++------- packages/backend-client/src/client.ts | 588 ++++++- 40 files changed, 9021 insertions(+), 1069 deletions(-) create mode 100644 backend/drizzle/0020_add-fail-on-trigger-rules.sql create mode 100644 backend/drizzle/0021_widen-github-ids-to-bigint.sql create mode 100644 backend/drizzle/0022_add-pr-review-fields.sql create mode 100644 backend/src/config/github-app.config.ts create mode 100644 backend/src/database/schema/github-app.ts create mode 100644 backend/src/github-app/__tests__/diff-hunk-parser.spec.ts create mode 100644 backend/src/github-app/__tests__/github-app.service.spec.ts create mode 100644 backend/src/github-app/__tests__/scan-result-sync.service.spec.ts create mode 100644 backend/src/github-app/diff-hunk-parser.ts create mode 100644 backend/src/github-app/dto/github-app.dto.ts create mode 100644 backend/src/github-app/github-app.controller.ts create mode 100644 backend/src/github-app/github-app.module.ts create mode 100644 backend/src/github-app/github-app.repository.ts create mode 100644 backend/src/github-app/github-app.service.ts create mode 100644 backend/src/github-app/index.ts create mode 100644 backend/src/github-app/scan-result-sync.service.ts create mode 100644 docs/GITHUB-SECURITY-WORKFLOWS.md create mode 100644 docs/samples/01-opengrep-repo-scan.json create mode 100644 docs/samples/02-trufflehog-secret-scan.json create mode 100644 docs/samples/03-security-scan-with-pr-comments.json create mode 100644 docs/samples/04-pr-security-scan.json create mode 100644 docs/samples/05-pr-security-check-run.json create mode 100644 docs/samples/06-opengrep-pr-repo-scan.json create mode 100644 docs/samples/07-ai-code-review.json create mode 100644 docs/samples/07-opengrep-pr-trigger.json create mode 100644 docs/samples/08-unified-ai-security-review.json diff --git a/.gitignore b/.gitignore index 494695ba3..725fedfdc 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,9 @@ build/ *.local *.tsbuildinfo +# Context dumps (may contain secrets) +.context + # Environment variables .env .env.local @@ -18,8 +21,7 @@ docker/.env .env.development.local .env.test.local .env.production.local -.env.eng-104 -.env.eng-104 +.env.e2e .shipsec-instance # Logs @@ -71,19 +73,6 @@ vite.config.ts.timestamp-* .playground/ .playground/ .playground/ +.playwright-mcp/ .omc/ MCP_FLOW_TRACE.md - -# Terraform / OpenTofu -.terraform/ -*.tfstate -*.tfstate.* -*.tfvars -*.tfvars.json -crash.log -crash.*.log -override.tf -override.tf.json -*_override.tf -*_override.tf.json -.terraform.lock.hcl.bak diff --git a/backend/.env.docker b/backend/.env.docker index 02bf7d6aa..633067b88 100644 --- a/backend/.env.docker +++ b/backend/.env.docker @@ -31,9 +31,11 @@ LOKI_PASSWORD="" # Kafka / Redpanda configuration for node I/O, log, and event ingestion (Docker network) LOG_KAFKA_BROKERS="redpanda:9092" - -# GitHub template repository configuration -GITHUB_TEMPLATE_REPO=shipsecai/workflow-templates -GITHUB_TEMPLATE_BRANCH=main -# Optional: GitHub personal access token for higher rate limits (60/hr → 5000/hr) -GITHUB_TEMPLATE_TOKEN= \ No newline at end of file +# GitHub App configuration (required for GitHub integration) +GITHUB_APP_ID="" +GITHUB_APP_PRIVATE_KEY="" +GITHUB_APP_WEBHOOK_SECRET="" +GITHUB_APP_CLIENT_ID="" +GITHUB_APP_CLIENT_SECRET="" +# GITHUB_API_URL="https://api.github.com" +# GITHUB_URL="https://github.com" \ No newline at end of file diff --git a/backend/.env.example b/backend/.env.example index e2c3f27b1..b63e5f3e2 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -68,3 +68,26 @@ REDIS_URL="" # Kafka / Redpanda configuration for node I/O, log, and event ingestion LOG_KAFKA_BROKERS="localhost:19092" + +# GitHub App configuration (required for GitHub integration) +# Create a GitHub App at https://github.com/settings/apps +GITHUB_APP_ID="" +# Private key in PEM format or base64-encoded PEM +# Generate from GitHub App settings > Private keys > Generate a private key +GITHUB_APP_PRIVATE_KEY="" +# Webhook secret configured in GitHub App settings +GITHUB_APP_WEBHOOK_SECRET="" +# OAuth credentials from GitHub App settings (optional, for OAuth flows) +GITHUB_APP_CLIENT_ID="" +GITHUB_APP_CLIENT_SECRET="" +# Override for GitHub Enterprise (defaults to public GitHub) +# GITHUB_API_URL="https://api.github.com" +# GITHUB_URL="https://github.com" + +# GitHub OAuth integration (optional, for user-level GitHub access) +# GITHUB_OAUTH_CLIENT_ID="" +# GITHUB_OAUTH_CLIENT_SECRET="" +# GITHUB_OAUTH_SCOPES="repo,read:user" + +# Webhook base URL (used for constructing callback URLs) +# WEBHOOK_BASE_URL="https://api.shipsec.ai" diff --git a/backend/drizzle/0020_add-fail-on-trigger-rules.sql b/backend/drizzle/0020_add-fail-on-trigger-rules.sql new file mode 100644 index 000000000..c7cd0aff4 --- /dev/null +++ b/backend/drizzle/0020_add-fail-on-trigger-rules.sql @@ -0,0 +1,5 @@ +-- Migration: Add fail_on column to github_trigger_rules +-- Configures the severity threshold at which a check run should report failure + +ALTER TABLE github_trigger_rules +ADD COLUMN fail_on VARCHAR(32) NOT NULL DEFAULT 'high'; diff --git a/backend/drizzle/0021_widen-github-ids-to-bigint.sql b/backend/drizzle/0021_widen-github-ids-to-bigint.sql new file mode 100644 index 000000000..72cebdd39 --- /dev/null +++ b/backend/drizzle/0021_widen-github-ids-to-bigint.sql @@ -0,0 +1,4 @@ +-- Widen check_run_id and pr_comment_id to bigint +-- GitHub IDs can exceed INT4 max (2,147,483,647) +ALTER TABLE "github_scan_results" ALTER COLUMN "check_run_id" SET DATA TYPE bigint; +ALTER TABLE "github_scan_results" ALTER COLUMN "pr_comment_id" SET DATA TYPE bigint; diff --git a/backend/drizzle/0022_add-pr-review-fields.sql b/backend/drizzle/0022_add-pr-review-fields.sql new file mode 100644 index 000000000..5aa217c45 --- /dev/null +++ b/backend/drizzle/0022_add-pr-review-fields.sql @@ -0,0 +1,6 @@ +-- Add post_pr_review rule toggle and pr_review_id result linkage +ALTER TABLE github_trigger_rules +ADD COLUMN post_pr_review BOOLEAN NOT NULL DEFAULT false; + +ALTER TABLE github_scan_results +ADD COLUMN pr_review_id bigint; diff --git a/backend/drizzle/meta/_journal.json b/backend/drizzle/meta/_journal.json index 761f469da..db7269ee3 100644 --- a/backend/drizzle/meta/_journal.json +++ b/backend/drizzle/meta/_journal.json @@ -96,8 +96,22 @@ { "idx": 13, "version": "7", - "when": 1762992000000, - "tag": "0026_add-run-status-cache", + "when": 1738972800000, + "tag": "0020_add-fail-on-trigger-rules", + "breakpoints": true + }, + { + "idx": 14, + "version": "7", + "when": 1739059200000, + "tag": "0021_widen-github-ids-to-bigint", + "breakpoints": true + }, + { + "idx": 15, + "version": "7", + "when": 1762646400000, + "tag": "0022_add-pr-review-fields", "breakpoints": true } ] diff --git a/backend/src/app.module.ts b/backend/src/app.module.ts index f7b24bf79..705a52c8d 100644 --- a/backend/src/app.module.ts +++ b/backend/src/app.module.ts @@ -10,7 +10,6 @@ import { AppController } from './app.controller'; import { AppService } from './app.service'; import { authConfig } from './config/auth.config'; import { opensearchConfig } from './config/opensearch.config'; -import { validateBackendEnv } from './config/env.validate'; import { OpenSearchModule } from './config/opensearch.module'; import { AgentsModule } from './agents/agents.module'; import { AuthModule } from './auth/auth.module'; @@ -26,15 +25,13 @@ import { IntegrationsModule } from './integrations/integrations.module'; import { SchedulesModule } from './schedules/schedules.module'; import { AnalyticsModule } from './analytics/analytics.module'; import { McpModule } from './mcp/mcp.module'; -import { StudioMcpModule } from './studio-mcp/studio-mcp.module'; -import { AuditModule } from './audit/audit.module'; import { ApiKeysModule } from './api-keys/api-keys.module'; import { WebhooksModule } from './webhooks/webhooks.module'; import { HumanInputsModule } from './human-inputs/human-inputs.module'; import { McpServersModule } from './mcp-servers/mcp-servers.module'; import { McpGroupsModule } from './mcp-groups/mcp-groups.module'; -import { TemplatesModule } from './templates/templates.module'; +import { GitHubAppModule } from './github-app/github-app.module'; const coreModules = [ AgentsModule, @@ -53,9 +50,7 @@ const coreModules = [ McpServersModule, McpGroupsModule, McpModule, - StudioMcpModule, - TemplatesModule, - AuditModule, + GitHubAppModule, ]; const testingModules = process.env.NODE_ENV === 'production' ? [] : [TestingSupportModule]; @@ -81,7 +76,6 @@ function getEnvFilePaths(): string[] { isGlobal: true, envFilePath: getEnvFilePaths(), load: [authConfig, opensearchConfig], - validate: validateBackendEnv, }), ThrottlerModule.forRootAsync({ useFactory: () => { diff --git a/backend/src/components/utils/categorization.ts b/backend/src/components/utils/categorization.ts index c52fb5025..4a9b37788 100644 --- a/backend/src/components/utils/categorization.ts +++ b/backend/src/components/utils/categorization.ts @@ -1,9 +1,4 @@ -import type { ComponentDefinition } from '@shipsec/component-sdk'; -import { - type ComponentCategory, - getComponentCategoryDescriptor, - normalizeComponentCategory, -} from '@shipsec/shared'; +import type { ComponentDefinition, ComponentCategory } from '@shipsec/component-sdk'; export interface ComponentCategoryConfig { label: string; @@ -13,13 +8,113 @@ export interface ComponentCategoryConfig { icon: string; } +const SUPPORTED_CATEGORIES: readonly ComponentCategory[] = [ + 'input', + 'transform', + 'ai', + 'mcp', + 'security', + 'scanners', + 'it_ops', + 'notification', + 'manual_action', + 'output', +]; + +const COMPONENT_CATEGORY_CONFIG: Record = { + input: { + label: 'Input', + color: 'text-blue-600', + description: 'Data sources, triggers, and credential access', + emoji: '📥', + icon: 'Download', + }, + transform: { + label: 'Transform', + color: 'text-orange-600', + description: 'Data processing, text manipulation, and formatting', + emoji: '🔄', + icon: 'RefreshCw', + }, + ai: { + label: 'AI Components', + color: 'text-violet-600', + description: 'AI-powered analysis and generation tools', + emoji: '🤖', + icon: 'Brain', + }, + mcp: { + label: 'MCP Servers', + color: 'text-teal-600', + description: 'Model Context Protocol servers and tool gateways', + emoji: '🔌', + icon: 'Plug', + }, + security: { + label: 'Security Tools', + color: 'text-red-600', + description: 'Security scanning and assessment tools', + emoji: '🔒', + icon: 'Shield', + }, + scanners: { + label: 'Scanners', + color: 'text-rose-600', + description: 'Code and secret scanning tools', + emoji: '🔍', + icon: 'Search', + }, + it_ops: { + label: 'IT Ops', + color: 'text-cyan-600', + description: 'IT operations and user management workflows', + emoji: '🏢', + icon: 'Building', + }, + notification: { + label: 'Notification', + color: 'text-pink-600', + description: 'Slack, Email, and other messaging alerts', + emoji: '🔔', + icon: 'Bell', + }, + manual_action: { + label: 'Manual Action', + color: 'text-amber-600', + description: 'Human-in-the-loop interactions, approvals, and manual tasks', + emoji: '👤', + icon: 'UserCheck', + }, + output: { + label: 'Output', + color: 'text-green-600', + description: 'Data export, notifications, and integrations', + emoji: '📤', + icon: 'Upload', + }, +}; + +function normalizeCategory(category?: string | null): ComponentCategory | null { + if (!category) { + return null; + } + + const normalized = category.toLowerCase(); + + if (SUPPORTED_CATEGORIES.includes(normalized as ComponentCategory)) { + return normalized as ComponentCategory; + } + + return null; +} + export function categorizeComponent(component: ComponentDefinition): ComponentCategory { - const fromMetadata = normalizeComponentCategory(component.ui?.category); + const fromMetadata = normalizeCategory(component.ui?.category); if (fromMetadata) { return fromMetadata; } - const fromDefinition = normalizeComponentCategory(component.category); + const fromDefinition = normalizeCategory(component.category); if (fromDefinition) { return fromDefinition; } @@ -28,12 +123,5 @@ export function categorizeComponent(component: ComponentDefinition): ComponentCa } export function getCategoryConfig(category: ComponentCategory): ComponentCategoryConfig { - const descriptor = getComponentCategoryDescriptor(category); - return { - label: descriptor.label, - color: descriptor.color, - description: descriptor.description, - emoji: descriptor.emoji, - icon: descriptor.icon, - }; + return COMPONENT_CATEGORY_CONFIG[category]; } diff --git a/backend/src/config/github-app.config.ts b/backend/src/config/github-app.config.ts new file mode 100644 index 000000000..4f2db3076 --- /dev/null +++ b/backend/src/config/github-app.config.ts @@ -0,0 +1,67 @@ +import { registerAs } from '@nestjs/config'; + +export interface GitHubAppConfig { + appId: string | null; + privateKey: string | null; + webhookSecret: string | null; + clientId: string | null; + clientSecret: string | null; + /** Base URL for GitHub API (default: https://api.github.com) */ + apiBaseUrl: string; + /** Base URL for GitHub (default: https://github.com) */ + baseUrl: string; +} + +export const githubAppConfig = registerAs('githubApp', () => { + // Private key can be provided as base64 or raw PEM + let privateKey = process.env.GITHUB_APP_PRIVATE_KEY ?? null; + if (privateKey && !privateKey.includes('-----BEGIN')) { + // Assume base64 encoded + try { + privateKey = Buffer.from(privateKey, 'base64').toString('utf-8'); + } catch { + // Keep as-is if decoding fails + } + } + + return { + appId: process.env.GITHUB_APP_ID ?? null, + privateKey, + webhookSecret: process.env.GITHUB_APP_WEBHOOK_SECRET ?? null, + clientId: process.env.GITHUB_APP_CLIENT_ID ?? null, + clientSecret: process.env.GITHUB_APP_CLIENT_SECRET ?? null, + apiBaseUrl: process.env.GITHUB_API_URL ?? 'https://api.github.com', + baseUrl: process.env.GITHUB_URL ?? 'https://github.com', + }; +}); + +export function isGitHubAppConfigured(config: GitHubAppConfig): boolean { + return Boolean(config.appId && config.privateKey); +} + +/** + * Get the GitHub App configuration from environment variables. + * This is useful for accessing the config outside of the NestJS DI context. + * For use within NestJS modules, prefer injecting ConfigService and using + * configService.get('githubApp'). + */ +export function getGitHubAppConfig(): GitHubAppConfig { + let privateKey = process.env.GITHUB_APP_PRIVATE_KEY ?? null; + if (privateKey && !privateKey.includes('-----BEGIN')) { + try { + privateKey = Buffer.from(privateKey, 'base64').toString('utf-8'); + } catch { + // Keep as-is if decoding fails + } + } + + return { + appId: process.env.GITHUB_APP_ID ?? null, + privateKey, + webhookSecret: process.env.GITHUB_APP_WEBHOOK_SECRET ?? null, + clientId: process.env.GITHUB_APP_CLIENT_ID ?? null, + clientSecret: process.env.GITHUB_APP_CLIENT_SECRET ?? null, + apiBaseUrl: process.env.GITHUB_API_URL ?? 'https://api.github.com', + baseUrl: process.env.GITHUB_URL ?? 'https://github.com', + }; +} diff --git a/backend/src/database/schema/github-app.ts b/backend/src/database/schema/github-app.ts new file mode 100644 index 000000000..e6828ae64 --- /dev/null +++ b/backend/src/database/schema/github-app.ts @@ -0,0 +1,252 @@ +import { + bigint, + boolean, + index, + integer, + jsonb, + pgTable, + text, + timestamp, + uniqueIndex, + uuid, + varchar, +} from 'drizzle-orm/pg-core'; + +/** + * GitHub App installations - stores info about orgs/users that installed the app + */ +export const githubAppInstallations = pgTable( + 'github_app_installations', + { + id: uuid('id').primaryKey().defaultRandom(), + /** GitHub's installation ID */ + installationId: integer('installation_id').notNull(), + /** 'Organization' or 'User' */ + accountType: varchar('account_type', { length: 32 }).notNull(), + /** GitHub username or org name */ + accountLogin: varchar('account_login', { length: 191 }).notNull(), + /** GitHub account ID */ + accountId: integer('account_id').notNull(), + /** Avatar URL */ + accountAvatarUrl: text('account_avatar_url'), + /** Permissions granted to the app */ + permissions: jsonb('permissions').$type>().default({}), + /** 'all' or 'selected' */ + repositorySelection: varchar('repository_selection', { length: 32 }).default('selected'), + /** ShipSec organization that owns this installation */ + organizationId: varchar('organization_id', { length: 191 }).notNull(), + /** User who initiated the installation */ + installedBy: varchar('installed_by', { length: 191 }), + /** Whether the installation is active */ + isActive: boolean('is_active').default(true).notNull(), + /** Suspended at timestamp (if suspended by GitHub) */ + suspendedAt: timestamp('suspended_at', { withTimezone: true }), + createdAt: timestamp('created_at', { withTimezone: true }).defaultNow().notNull(), + updatedAt: timestamp('updated_at', { withTimezone: true }).defaultNow().notNull(), + }, + (table) => ({ + installationIdIdx: uniqueIndex('github_installations_installation_id_uidx').on( + table.installationId, + ), + orgIdx: index('github_installations_org_idx').on(table.organizationId), + accountLoginIdx: index('github_installations_account_login_idx').on(table.accountLogin), + }), +); + +/** + * GitHub repositories connected via installations + */ +export const githubRepositories = pgTable( + 'github_repositories', + { + id: uuid('id').primaryKey().defaultRandom(), + /** Reference to the installation */ + installationId: uuid('installation_id') + .notNull() + .references(() => githubAppInstallations.id, { onDelete: 'cascade' }), + /** GitHub's repository ID */ + repoId: integer('repo_id').notNull(), + /** Full name like 'owner/repo' */ + fullName: varchar('full_name', { length: 255 }).notNull(), + /** Repository name only */ + name: varchar('name', { length: 191 }).notNull(), + /** Owner login */ + owner: varchar('owner', { length: 191 }).notNull(), + /** Whether the repo is private */ + isPrivate: boolean('is_private').default(false).notNull(), + /** Default branch name */ + defaultBranch: varchar('default_branch', { length: 191 }).default('main'), + /** Repository description */ + description: text('description'), + /** Primary language */ + language: varchar('language', { length: 64 }), + /** Clone URL (https) */ + cloneUrl: text('clone_url'), + /** HTML URL */ + htmlUrl: text('html_url'), + /** ShipSec organization */ + organizationId: varchar('organization_id', { length: 191 }).notNull(), + /** Whether scans are enabled for this repo */ + scansEnabled: boolean('scans_enabled').default(true).notNull(), + /** Last synced from GitHub */ + lastSyncedAt: timestamp('last_synced_at', { withTimezone: true }), + createdAt: timestamp('created_at', { withTimezone: true }).defaultNow().notNull(), + updatedAt: timestamp('updated_at', { withTimezone: true }).defaultNow().notNull(), + }, + (table) => ({ + repoIdIdx: uniqueIndex('github_repos_repo_id_uidx').on(table.repoId), + installationIdx: index('github_repos_installation_idx').on(table.installationId), + orgIdx: index('github_repos_org_idx').on(table.organizationId), + fullNameIdx: index('github_repos_full_name_idx').on(table.fullName), + }), +); + +/** + * Trigger rules for automated workflows on GitHub events + */ +export const githubTriggerRules = pgTable( + 'github_trigger_rules', + { + id: uuid('id').primaryKey().defaultRandom(), + /** Human-readable name */ + name: varchar('name', { length: 191 }).notNull(), + /** Description of what this rule does */ + description: text('description'), + /** Pattern to match repos: 'org/repo', 'org/*', or '*' */ + repositoryPattern: varchar('repository_pattern', { length: 255 }).notNull(), + /** Event type: 'pull_request', 'push', 'release', 'repository_added' */ + event: varchar('event', { length: 64 }).notNull(), + /** PR actions to trigger on: ['opened', 'synchronize', 'reopened'] */ + actions: jsonb('actions').$type().default([]), + /** Branch patterns to match: ['main', 'release/*'] */ + branches: jsonb('branches').$type().default([]), + /** Workflow ID to trigger */ + workflowId: uuid('workflow_id').notNull(), + /** Whether to post results as PR comment */ + postPrComment: boolean('post_pr_comment').default(true).notNull(), + /** Whether to create GitHub Check Run */ + createCheckRun: boolean('create_check_run').default(true).notNull(), + /** Whether to post inline PR review comments */ + postPrReview: boolean('post_pr_review').default(false).notNull(), + /** Severity threshold for failing the check run: 'critical','high','medium','low','info','none' */ + failOn: varchar('fail_on', { length: 32 }).default('high').notNull(), + /** Whether this rule is enabled */ + enabled: boolean('enabled').default(true).notNull(), + /** Priority for rule matching (lower = higher priority) */ + priority: integer('priority').default(100).notNull(), + /** ShipSec organization */ + organizationId: varchar('organization_id', { length: 191 }).notNull(), + /** User who created the rule */ + createdBy: varchar('created_by', { length: 191 }), + /** Soft delete timestamp */ + deletedAt: timestamp('deleted_at', { withTimezone: true }), + createdAt: timestamp('created_at', { withTimezone: true }).defaultNow().notNull(), + updatedAt: timestamp('updated_at', { withTimezone: true }).defaultNow().notNull(), + }, + (table) => ({ + orgIdx: index('github_trigger_rules_org_idx').on(table.organizationId), + eventIdx: index('github_trigger_rules_event_idx').on(table.event), + enabledIdx: index('github_trigger_rules_enabled_idx').on(table.enabled), + }), +); + +/** + * Scan results linked to GitHub PRs/commits + */ +export const githubScanResults = pgTable( + 'github_scan_results', + { + id: uuid('id').primaryKey().defaultRandom(), + /** Reference to the repository */ + repositoryId: uuid('repository_id') + .notNull() + .references(() => githubRepositories.id, { onDelete: 'cascade' }), + /** ShipSec workflow run ID */ + workflowRunId: varchar('workflow_run_id', { length: 191 }).notNull(), + /** Source type: 'pr', 'push', 'manual', 'schedule' */ + sourceType: varchar('source_type', { length: 32 }).notNull(), + /** PR number (if PR scan) */ + prNumber: integer('pr_number'), + /** Branch name */ + branch: varchar('branch', { length: 191 }), + /** Commit SHA */ + commitSha: varchar('commit_sha', { length: 64 }), + /** Scan status: 'pending', 'running', 'success', 'failure', 'error' */ + status: varchar('status', { length: 32 }).notNull().default('pending'), + /** Summary of findings */ + summary: jsonb('summary') + .$type<{ + critical: number; + high: number; + medium: number; + low: number; + info: number; + }>() + .default({ critical: 0, high: 0, medium: 0, low: 0, info: 0 }), + /** Total findings count */ + findingsCount: integer('findings_count').default(0).notNull(), + /** Detailed findings from the scan */ + findings: jsonb('findings') + .$type< + { + id: string; + type: string; + severity: 'critical' | 'high' | 'medium' | 'low' | 'info'; + file: string; + line?: number; + endLine?: number; + message: string; + snippet?: string; + ruleId?: string; + category?: string; + }[] + >() + .default([]), + /** GitHub Check Run ID (if created) */ + checkRunId: bigint('check_run_id', { mode: 'number' }), + /** GitHub PR comment ID (if posted) */ + prCommentId: bigint('pr_comment_id', { mode: 'number' }), + /** GitHub PR review ID (if posted) */ + prReviewId: bigint('pr_review_id', { mode: 'number' }), + /** External URL to full results */ + resultsUrl: text('results_url'), + /** Error message if scan failed */ + errorMessage: text('error_message'), + /** Trigger rule that initiated this scan */ + triggerRuleId: uuid('trigger_rule_id').references(() => githubTriggerRules.id, { + onDelete: 'set null', + }), + /** ShipSec organization */ + organizationId: varchar('organization_id', { length: 191 }).notNull(), + /** Started at */ + startedAt: timestamp('started_at', { withTimezone: true }), + /** Completed at */ + completedAt: timestamp('completed_at', { withTimezone: true }), + createdAt: timestamp('created_at', { withTimezone: true }).defaultNow().notNull(), + updatedAt: timestamp('updated_at', { withTimezone: true }).defaultNow().notNull(), + }, + (table) => ({ + repoIdx: index('github_scan_results_repo_idx').on(table.repositoryId), + orgIdx: index('github_scan_results_org_idx').on(table.organizationId), + prIdx: index('github_scan_results_pr_idx').on(table.repositoryId, table.prNumber), + statusIdx: index('github_scan_results_status_idx').on(table.status), + workflowRunIdx: index('github_scan_results_workflow_run_idx').on(table.workflowRunId), + triggerRuleIdx: index('github_scan_results_trigger_rule_idx').on( + table.triggerRuleId, + table.createdAt, + ), + }), +); + +// Type exports +export type GitHubAppInstallationRecord = typeof githubAppInstallations.$inferSelect; +export type NewGitHubAppInstallationRecord = typeof githubAppInstallations.$inferInsert; + +export type GitHubRepositoryRecord = typeof githubRepositories.$inferSelect; +export type NewGitHubRepositoryRecord = typeof githubRepositories.$inferInsert; + +export type GitHubTriggerRuleRecord = typeof githubTriggerRules.$inferSelect; +export type NewGitHubTriggerRuleRecord = typeof githubTriggerRules.$inferInsert; + +export type GitHubScanResultRecord = typeof githubScanResults.$inferSelect; +export type NewGitHubScanResultRecord = typeof githubScanResults.$inferInsert; diff --git a/backend/src/database/schema/index.ts b/backend/src/database/schema/index.ts index 89fec502d..5ae926740 100644 --- a/backend/src/database/schema/index.ts +++ b/backend/src/database/schema/index.ts @@ -13,12 +13,10 @@ export * from './integrations'; export * from './workflow-schedules'; export * from './human-input-requests'; export * from './webhooks'; -export * from './audit-logs'; export * from './terminal-records'; export * from './agent-trace-events'; export * from './mcp-servers'; export * from './node-io'; -export * from './organization-settings'; -export * from './templates'; +export * from './github-app'; diff --git a/backend/src/dsl/validator.ts b/backend/src/dsl/validator.ts index 5df2437cd..0376600f1 100644 --- a/backend/src/dsl/validator.ts +++ b/backend/src/dsl/validator.ts @@ -161,34 +161,30 @@ function isPlaceholderIssue(issue: ZodIssue, placeholderFields: Set): bo return false; } - switch (issue.code) { - case 'invalid_type': - return true; - case 'invalid_format': - return true; - case 'too_small': - return true; - case 'too_big': - return true; - case 'invalid_value': - // Enum/literal validation fails on placeholder objects with missing fields - // The actual value from upstream will have the correct enum value at runtime - return true; - case 'custom': - // Custom validations (from .refine()) fail on placeholders but will pass at runtime - // when the actual value comes from the connected edge - return true; - case 'invalid_union': - if ('unionErrors' in issue) { - const unionIssue = issue as ZodIssue & { unionErrors: ZodError[] }; - return unionIssue.unionErrors.every((variant: ZodError) => - variant.issues.every((inner) => inner.code === 'invalid_type'), - ); - } - return false; - default: - return false; + const code = String(issue.code); + const placeholderCodes = new Set([ + 'invalid_type', + 'invalid_format', + 'invalid_enum_value', // Zod v3 + 'invalid_value', // Zod v4 (enum / literal validation) + 'invalid_string', // Zod v3 + 'too_small', + 'too_big', + 'custom', + ]); + + if (placeholderCodes.has(code)) { + return true; } + + if (code === 'invalid_union' && 'unionErrors' in issue) { + const unionIssue = issue as ZodIssue & { unionErrors: ZodError[] }; + return unionIssue.unionErrors.every((variant: ZodError) => + variant.issues.every((inner) => placeholderCodes.has(String(inner.code))), + ); + } + + return false; } /** diff --git a/backend/src/github-app/__tests__/diff-hunk-parser.spec.ts b/backend/src/github-app/__tests__/diff-hunk-parser.spec.ts new file mode 100644 index 000000000..edc8a7d5e --- /dev/null +++ b/backend/src/github-app/__tests__/diff-hunk-parser.spec.ts @@ -0,0 +1,84 @@ +import { describe, expect, it } from 'bun:test'; +import { + buildPatchMap, + escapeBackticks, + escapeMarkdown, + isLineInDiffHunks, + parseDiffHunks, + safeTruncate, + stripWorkspacePrefix, +} from '../diff-hunk-parser'; + +describe('diff-hunk-parser', () => { + describe('parseDiffHunks', () => { + it('parses single and multi hunk patches', () => { + const patch = ['@@ -1,2 +10,4 @@', ' a', '+b', '@@ -20,1 +40,2 @@', ' c'].join('\n'); + expect(parseDiffHunks(patch)).toEqual([ + { newStart: 10, newCount: 4 }, + { newStart: 40, newCount: 2 }, + ]); + }); + + it('defaults count to 1 when omitted', () => { + expect(parseDiffHunks('@@ -5 +8 @@\n+x')).toEqual([{ newStart: 8, newCount: 1 }]); + }); + + it('returns empty on empty patch', () => { + expect(parseDiffHunks('')).toEqual([]); + }); + }); + + describe('isLineInDiffHunks', () => { + const patch = ['@@ -1,1 +10,3 @@', ' a', '+b', '@@ -30,1 +40,2 @@', ' c'].join('\n'); + + it('matches lines inside a hunk range', () => { + expect(isLineInDiffHunks(10, patch)).toBe(true); + expect(isLineInDiffHunks(12, patch)).toBe(true); + expect(isLineInDiffHunks(40, patch)).toBe(true); + expect(isLineInDiffHunks(41, patch)).toBe(true); + }); + + it('rejects lines outside hunk ranges', () => { + expect(isLineInDiffHunks(9, patch)).toBe(false); + expect(isLineInDiffHunks(13, patch)).toBe(false); + expect(isLineInDiffHunks(39, patch)).toBe(false); + expect(isLineInDiffHunks(42, patch)).toBe(false); + expect(isLineInDiffHunks(1, '')).toBe(false); + }); + }); + + describe('buildPatchMap', () => { + it('builds patch map and skips files without patch', () => { + const patchMap = buildPatchMap([ + { filename: 'src/a.ts', patch: '@@ -1 +1 @@' }, + { filename: 'src/b.ts' }, + ]); + expect(patchMap.size).toBe(1); + expect(patchMap.get('src/a.ts')).toBe('@@ -1 +1 @@'); + expect(patchMap.get('src/b.ts')).toBeUndefined(); + }); + }); + + describe('stripWorkspacePrefix', () => { + it('strips known prefixes and leading slash', () => { + expect(stripWorkspacePrefix('/workspace/repo/src/a.ts')).toBe('src/a.ts'); + expect(stripWorkspacePrefix('/scan/repo/src/b.ts')).toBe('src/b.ts'); + expect(stripWorkspacePrefix('/repo/src/c.ts')).toBe('src/c.ts'); + expect(stripWorkspacePrefix('/src/d.ts')).toBe('src/d.ts'); + expect(stripWorkspacePrefix('src/e.ts')).toBe('src/e.ts'); + }); + }); + + describe('escaping helpers', () => { + it('escapes markdown chars and inner backticks', () => { + expect(escapeMarkdown('')).toBe('<a\\|b>'); + expect(escapeBackticks('a`b`c')).toBe('a``b``c'); + }); + + it('truncates safely and avoids dangling code fences', () => { + const long = ['line 1', '```', 'line 2', 'line 3'].join('\n'); + const truncated = safeTruncate(long, 14); + expect(truncated.endsWith('\n...')).toBe(true); + }); + }); +}); diff --git a/backend/src/github-app/__tests__/github-app.service.spec.ts b/backend/src/github-app/__tests__/github-app.service.spec.ts new file mode 100644 index 000000000..81f47487b --- /dev/null +++ b/backend/src/github-app/__tests__/github-app.service.spec.ts @@ -0,0 +1,222 @@ +import { beforeEach, describe, expect, it, vi } from 'bun:test'; +import { GitHubAppService } from '../github-app.service'; +import type { AuthContext } from '../../auth/types'; + +function makeRuleRecord(overrides: Record = {}) { + return { + id: 'rule-1', + name: 'Rule', + description: null, + repositoryPattern: 'org/*', + event: 'pull_request', + actions: ['opened'], + branches: ['main'], + workflowId: '11111111-1111-1111-1111-111111111111', + postPrComment: true, + createCheckRun: true, + postPrReview: false, + failOn: 'high', + enabled: true, + priority: 100, + organizationId: 'org-1', + createdBy: 'user-1', + createdAt: new Date(), + updatedAt: new Date(), + ...overrides, + }; +} + +function makeScanRecord(overrides: Record = {}) { + const now = new Date('2026-02-10T05:00:00.000Z'); + return { + id: 'scan-1', + repositoryId: 'repo-1', + workflowRunId: 'run-1', + sourceType: 'manual', + prNumber: null, + branch: 'main', + commitSha: null, + status: 'failure', + summary: { critical: 0, high: 0, medium: 0, low: 0, info: 0 }, + findingsCount: 0, + findings: [], + checkRunId: null, + prCommentId: null, + prReviewId: null, + resultsUrl: null, + errorMessage: 'Missing PR context: prNumber not provided', + triggerRuleId: null, + organizationId: 'org-1', + startedAt: now, + completedAt: null, + createdAt: now, + updatedAt: now, + workflowName: 'Unified AI Security Review', + workflowVersion: 4, + triggerType: 'schedule', + triggerSource: 'sched-1', + triggerLabel: 'Nightly Dependency Audit', + ...overrides, + }; +} + +describe('GitHubAppService', () => { + let service: GitHubAppService; + let repository: Record>; + const auth: AuthContext = { + userId: 'user-1', + organizationId: 'org-1', + roles: [], + isAuthenticated: true, + provider: 'test', + }; + + beforeEach(() => { + repository = { + createTriggerRule: vi.fn(), + updateTriggerRule: vi.fn(), + findTriggerRuleById: vi.fn(), + listScanResultsByOrg: vi.fn(), + }; + + service = new GitHubAppService( + { + get: vi.fn().mockReturnValue({ + appId: '1', + privateKey: 'dummy', + baseUrl: 'https://github.com', + apiBaseUrl: 'https://api.github.com', + }), + } as any, + repository as any, + {} as any, + ); + }); + + describe('trigger rule CRUD pass-through', () => { + it('defaults postPrReview to false on create when omitted', async () => { + repository.createTriggerRule.mockResolvedValue(makeRuleRecord({ postPrReview: false })); + + const result = await service.createTriggerRule( + { + name: 'Rule', + repositoryPattern: 'org/*', + event: 'pull_request', + workflowId: '11111111-1111-1111-1111-111111111111', + } as any, + auth, + ); + + expect(repository.createTriggerRule).toHaveBeenCalledWith( + expect.objectContaining({ postPrReview: false }), + ); + expect(result.postPrReview).toBe(false); + }); + + it('passes postPrReview true on create', async () => { + repository.createTriggerRule.mockResolvedValue(makeRuleRecord({ postPrReview: true })); + + const result = await service.createTriggerRule( + { + name: 'Rule', + repositoryPattern: 'org/*', + event: 'pull_request', + workflowId: '11111111-1111-1111-1111-111111111111', + postPrReview: true, + } as any, + auth, + ); + + expect(repository.createTriggerRule).toHaveBeenCalledWith( + expect.objectContaining({ postPrReview: true }), + ); + expect(result.postPrReview).toBe(true); + }); + + it('updates postPrReview when provided', async () => { + repository.findTriggerRuleById.mockResolvedValue( + makeRuleRecord({ id: 'rule-1', organizationId: 'org-1' }), + ); + repository.updateTriggerRule.mockResolvedValue(makeRuleRecord({ postPrReview: true })); + + const result = await service.updateTriggerRule('rule-1', { postPrReview: true } as any, auth); + + expect(repository.updateTriggerRule).toHaveBeenCalledWith( + 'rule-1', + expect.objectContaining({ postPrReview: true }), + ); + expect(result.postPrReview).toBe(true); + }); + }); + + describe('getPullRequestFiles pagination', () => { + it('returns one page when fewer than 100 files', async () => { + const apiSpy = vi.spyOn(service as any, 'githubApiRequest').mockResolvedValue([ + { filename: 'a.ts', status: 'modified' }, + { filename: 'b.ts', status: 'modified' }, + ]); + + const files = await service.getPullRequestFiles(1, 'org', 'repo', 42); + + expect(files).toHaveLength(2); + expect(apiSpy).toHaveBeenCalledTimes(1); + }); + + it('accumulates pages until a short page is returned', async () => { + const pageOne = Array.from({ length: 100 }, (_, i) => ({ + filename: `a-${i}.ts`, + status: 'modified', + })); + const pageTwo = [{ filename: 'last.ts', status: 'modified' }]; + const apiSpy = vi + .spyOn(service as any, 'githubApiRequest') + .mockResolvedValueOnce(pageOne) + .mockResolvedValueOnce(pageTwo); + + const files = await service.getPullRequestFiles(1, 'org', 'repo', 42); + + expect(files).toHaveLength(101); + expect(apiSpy).toHaveBeenCalledTimes(2); + }); + + it('stops at 30 pages', async () => { + const fullPage = Array.from({ length: 100 }, (_, i) => ({ + filename: `f-${i}.ts`, + status: 'modified', + })); + const apiSpy = vi.spyOn(service as any, 'githubApiRequest').mockResolvedValue(fullPage); + + const files = await service.getPullRequestFiles(1, 'org', 'repo', 42); + + expect(apiSpy).toHaveBeenCalledTimes(30); + expect(files).toHaveLength(3000); + }); + }); + + describe('scan result mapping', () => { + it('maps schedule metadata from workflow run trigger fields', async () => { + repository.listScanResultsByOrg.mockResolvedValue([makeScanRecord()]); + + const results = await service.listScanResults(auth, { + source: 'schedule', + scheduleId: 'sched-1', + triggerRuleId: 'rule-1', + }); + + expect(repository.listScanResultsByOrg).toHaveBeenCalledWith( + 'org-1', + expect.objectContaining({ + source: 'schedule', + scheduleId: 'sched-1', + triggerRuleId: 'rule-1', + }), + ); + expect(results).toHaveLength(1); + expect(results[0]?.sourceType).toBe('schedule'); + expect(results[0]?.scheduleId).toBe('sched-1'); + expect(results[0]?.scheduleName).toBe('Nightly Dependency Audit'); + expect(results[0]?.triggerType).toBe('schedule'); + expect(results[0]?.triggerSource).toBe('sched-1'); + }); + }); +}); diff --git a/backend/src/github-app/__tests__/scan-result-sync.service.spec.ts b/backend/src/github-app/__tests__/scan-result-sync.service.spec.ts new file mode 100644 index 000000000..eee3a09c3 --- /dev/null +++ b/backend/src/github-app/__tests__/scan-result-sync.service.spec.ts @@ -0,0 +1,1085 @@ +import { beforeEach, describe, expect, it, vi } from 'bun:test'; +import { ScanResultSyncService } from '../scan-result-sync.service'; +import type { GitHubAppRepository } from '../github-app.repository'; +import type { GitHubAppService } from '../github-app.service'; +import type { TemporalService } from '../../temporal/temporal.service'; +import type { GitHubScanResultRecord } from '../../database/schema'; + +function makeScanResult(overrides: Partial = {}): GitHubScanResultRecord { + return { + id: 'scan-1', + repositoryId: 'repo-1', + workflowRunId: 'shipsec-run-abc', + sourceType: 'pr', + prNumber: 42, + branch: 'main', + commitSha: 'abc123', + status: 'running', + summary: { critical: 0, high: 0, medium: 0, low: 0, info: 0 }, + findingsCount: 0, + findings: [], + checkRunId: null, + prCommentId: null, + prReviewId: null, + resultsUrl: null, + errorMessage: null, + triggerRuleId: null, + organizationId: 'org-1', + startedAt: new Date(), + completedAt: null, + createdAt: new Date(), + updatedAt: new Date(), + ...overrides, + }; +} + +describe('ScanResultSyncService', () => { + let service: ScanResultSyncService; + let mockRepository: { + findRunningScanResults: ReturnType; + updateScanResult: ReturnType; + findTriggerRuleById: ReturnType; + findRepositoryById: ReturnType; + findInstallationById: ReturnType; + }; + let mockTemporalService: { + describeWorkflow: ReturnType; + getWorkflowResult: ReturnType; + }; + let mockGitHubAppService: { + updateCheckRun: ReturnType; + getPullRequestFiles: ReturnType; + createPullRequestReview: ReturnType; + }; + + beforeEach(() => { + mockRepository = { + findRunningScanResults: vi.fn().mockResolvedValue([]), + updateScanResult: vi.fn().mockResolvedValue(undefined), + findTriggerRuleById: vi.fn().mockResolvedValue(null), + findRepositoryById: vi.fn().mockResolvedValue(null), + findInstallationById: vi.fn().mockResolvedValue(null), + }; + + mockTemporalService = { + describeWorkflow: vi.fn(), + getWorkflowResult: vi.fn(), + }; + + mockGitHubAppService = { + updateCheckRun: vi.fn().mockResolvedValue(undefined), + getPullRequestFiles: vi.fn().mockResolvedValue([]), + createPullRequestReview: vi.fn(), + }; + + service = new ScanResultSyncService( + mockRepository as unknown as GitHubAppRepository, + mockTemporalService as unknown as TemporalService, + mockGitHubAppService as unknown as GitHubAppService, + ); + }); + + // ============ Core sync behavior ============ + + it('should skip when no running scan results exist', async () => { + mockRepository.findRunningScanResults.mockResolvedValue([]); + await service.syncRunningScanResults(); + expect(mockTemporalService.describeWorkflow).not.toHaveBeenCalled(); + }); + + it('should skip scan results whose workflow is still running', async () => { + const scanResult = makeScanResult(); + mockRepository.findRunningScanResults.mockResolvedValue([scanResult]); + mockTemporalService.describeWorkflow.mockResolvedValue({ + workflowId: 'shipsec-run-abc', + runId: 'temporal-run-1', + status: 'RUNNING', + startTime: new Date().toISOString(), + }); + + await service.syncRunningScanResults(); + expect(mockRepository.updateScanResult).not.toHaveBeenCalled(); + }); + + it('should mark scan as success when workflow completes with findings', async () => { + const scanResult = makeScanResult(); + mockRepository.findRunningScanResults.mockResolvedValue([scanResult]); + mockTemporalService.describeWorkflow.mockResolvedValue({ + workflowId: 'shipsec-run-abc', + status: 'COMPLETED', + }); + mockTemporalService.getWorkflowResult.mockResolvedValue({ + outputs: { + 'normalize-1': { + summary: { critical: 1, high: 2, medium: 0, low: 0, info: 0 }, + findingCount: 3, + }, + 'check-1': { + checkRunId: 12345, + }, + 'comment-1': { + commentId: 67890, + }, + }, + success: true, + }); + + await service.syncRunningScanResults(); + + expect(mockRepository.updateScanResult).toHaveBeenCalledWith('scan-1', { + status: 'success', + summary: { critical: 1, high: 2, medium: 0, low: 0, info: 0 }, + findingsCount: 3, + findings: [], + checkRunId: 12345, + prCommentId: 67890, + completedAt: expect.any(Date), + }); + }); + + it('should merge findings across normalize nodes and map findings fields', async () => { + const scanResult = makeScanResult(); + mockRepository.findRunningScanResults.mockResolvedValue([scanResult]); + mockTemporalService.describeWorkflow.mockResolvedValue({ + workflowId: 'shipsec-run-abc', + status: 'COMPLETED', + }); + mockTemporalService.getWorkflowResult.mockResolvedValue({ + outputs: { + 'normalize-1': { + summary: { critical: 1, high: 1, medium: 0, low: 0, info: 0 }, + findingCount: 2, + findings: [ + { + finding_hash: 'th-1', + scanner: 'trufflehog', + severity: 'HIGH', + file: 'src/secrets.ts', + line: 7, + description: 'Detected AWS key', + snippet: 'AKIA....', + rule_id: 'aws_access_key', + }, + ], + }, + 'normalize-2': { + summary: { critical: 0, high: 0, medium: 1, low: 1, info: 0 }, + findings: [ + { + id: 'og-1', + scanner: 'opengrep', + severity: 'medium', + file: 'src/db.ts', + line: 12, + message: 'Potential SQL injection', + rule_id: 'sql-injection', + }, + { + id: 'misc-1', + type: 'custom-finding', + severity: 'unexpected', + file: 'README.md', + title: 'Unexpected output shape', + }, + { + id: 'empty-desc-1', + scanner: 'custom-scanner', + severity: 'low', + file: 'src/custom.ts', + description: ' ', + message: 'Fallback message should be preserved', + }, + ], + }, + 'check-1': { + checkRunId: 12345, + }, + 'comment-1': { + commentId: 67890, + }, + }, + success: true, + }); + + await service.syncRunningScanResults(); + + expect(mockRepository.updateScanResult).toHaveBeenCalledWith( + 'scan-1', + expect.objectContaining({ + status: 'success', + summary: { critical: 1, high: 1, medium: 1, low: 1, info: 0 }, + findingsCount: 4, + checkRunId: 12345, + prCommentId: 67890, + findings: expect.arrayContaining([ + expect.objectContaining({ + id: 'th-1', + type: 'trufflehog', + severity: 'high', + file: 'src/secrets.ts', + line: 7, + message: 'Detected AWS key', + ruleId: 'aws_access_key', + category: 'trufflehog', + }), + expect.objectContaining({ + id: 'og-1', + type: 'opengrep', + severity: 'medium', + file: 'src/db.ts', + line: 12, + message: 'Potential SQL injection', + ruleId: 'sql-injection', + category: 'opengrep', + }), + expect.objectContaining({ + id: 'misc-1', + type: 'custom-finding', + severity: 'info', + file: 'README.md', + message: 'Unexpected output shape', + }), + expect.objectContaining({ + id: 'empty-desc-1', + type: 'custom-scanner', + severity: 'low', + file: 'src/custom.ts', + message: 'Fallback message should be preserved', + }), + ]), + completedAt: expect.any(Date), + }), + ); + }); + + it('should mark scan as failure when result extraction fails', async () => { + const scanResult = makeScanResult(); + mockRepository.findRunningScanResults.mockResolvedValue([scanResult]); + mockTemporalService.describeWorkflow.mockResolvedValue({ + workflowId: 'shipsec-run-abc', + status: 'COMPLETED', + }); + mockTemporalService.getWorkflowResult.mockRejectedValue(new Error('result not found')); + + await service.syncRunningScanResults(); + + expect(mockRepository.updateScanResult).toHaveBeenCalledWith('scan-1', { + status: 'failure', + errorMessage: expect.stringContaining('result not found'), + completedAt: expect.any(Date), + }); + }); + + it('should mark scan as failure when workflow fails', async () => { + const scanResult = makeScanResult(); + mockRepository.findRunningScanResults.mockResolvedValue([scanResult]); + mockTemporalService.describeWorkflow.mockResolvedValue({ + workflowId: 'shipsec-run-abc', + status: 'FAILED', + failure: { message: 'Component timeout' }, + }); + + await service.syncRunningScanResults(); + + expect(mockRepository.updateScanResult).toHaveBeenCalledWith('scan-1', { + status: 'failure', + errorMessage: 'Component timeout', + completedAt: expect.any(Date), + }); + }); + + it('should mark scan as failure when workflow is cancelled', async () => { + const scanResult = makeScanResult(); + mockRepository.findRunningScanResults.mockResolvedValue([scanResult]); + mockTemporalService.describeWorkflow.mockResolvedValue({ + workflowId: 'shipsec-run-abc', + status: 'CANCELLED', + failure: undefined, + }); + + await service.syncRunningScanResults(); + + expect(mockRepository.updateScanResult).toHaveBeenCalledWith('scan-1', { + status: 'failure', + errorMessage: 'Workflow CANCELLED', + completedAt: expect.any(Date), + }); + }); + + it('should mark scan as failure when workflow not found in Temporal', async () => { + const scanResult = makeScanResult(); + mockRepository.findRunningScanResults.mockResolvedValue([scanResult]); + mockTemporalService.describeWorkflow.mockRejectedValue(new Error('not found')); + + await service.syncRunningScanResults(); + + expect(mockRepository.updateScanResult).toHaveBeenCalledWith('scan-1', { + status: 'failure', + errorMessage: 'Workflow run not found', + completedAt: expect.any(Date), + }); + }); + + it('should handle multiple scan results independently', async () => { + const scan1 = makeScanResult({ id: 'scan-1', workflowRunId: 'run-1' }); + const scan2 = makeScanResult({ id: 'scan-2', workflowRunId: 'run-2' }); + mockRepository.findRunningScanResults.mockResolvedValue([scan1, scan2]); + + mockTemporalService.describeWorkflow + .mockResolvedValueOnce({ workflowId: 'run-1', status: 'COMPLETED' }) + .mockResolvedValueOnce({ workflowId: 'run-2', status: 'RUNNING' }); + mockTemporalService.getWorkflowResult.mockResolvedValueOnce({ + outputs: {}, + success: true, + }); + + await service.syncRunningScanResults(); + + // First should be updated, second should not + expect(mockRepository.updateScanResult).toHaveBeenCalledTimes(1); + expect(mockRepository.updateScanResult).toHaveBeenCalledWith('scan-1', { + status: 'success', + completedAt: expect.any(Date), + }); + }); + + it('should allow commentId overwrite (last valid wins)', async () => { + const scanResult = makeScanResult(); + mockRepository.findRunningScanResults.mockResolvedValue([scanResult]); + mockTemporalService.describeWorkflow.mockResolvedValue({ + workflowId: 'shipsec-run-abc', + status: 'COMPLETED', + }); + mockTemporalService.getWorkflowResult.mockResolvedValue({ + outputs: { + 'comment-rule-a': { commentId: 111 }, + 'comment-rule-b': { commentId: 222 }, + }, + success: true, + }); + + await service.syncRunningScanResults(); + + expect(mockRepository.updateScanResult).toHaveBeenCalledWith( + 'scan-1', + expect.objectContaining({ prCommentId: 222 }), + ); + }); + + // ============ postPrReviewIfEnabled ============ + + describe('postPrReviewIfEnabled', () => { + const setupReviewContext = () => { + mockRepository.findRepositoryById.mockResolvedValue({ + id: 'repo-1', + installationId: 'install-1', + fullName: 'myorg/myrepo', + }); + mockRepository.findInstallationById.mockResolvedValue({ + id: 'install-1', + installationId: 99999, + }); + mockRepository.findTriggerRuleById.mockResolvedValue({ + id: 'rule-1', + failOn: 'high', + postPrReview: true, + }); + mockGitHubAppService.getPullRequestFiles.mockResolvedValue([ + { + filename: 'src/a.ts', + status: 'modified', + patch: '@@ -1,1 +1,6 @@\n context\n+new line', + }, + ]); + }; + + it('posts review with inline comments for findings in diff hunks', async () => { + setupReviewContext(); + mockGitHubAppService.createPullRequestReview.mockResolvedValue({ + id: 900, + html_url: 'https://github.com/myorg/myrepo/pull/1#pullrequestreview-900', + }); + + const scanResult = makeScanResult({ triggerRuleId: 'rule-1' }); + mockRepository.findRunningScanResults.mockResolvedValue([scanResult]); + mockTemporalService.describeWorkflow.mockResolvedValue({ + workflowId: 'shipsec-run-abc', + status: 'COMPLETED', + }); + mockTemporalService.getWorkflowResult.mockResolvedValue({ + outputs: { + normalize: { + summary: { critical: 0, high: 1, medium: 0, low: 0, info: 0 }, + findingCount: 2, + findings: [ + { + scanner: 'opengrep', + severity: 'high', + file: 'src/a.ts', + line: 2, + message: 'Inline issue', + rule_id: 'rule-a', + }, + { + scanner: 'opengrep', + severity: 'medium', + file: 'src/a.ts', + line: 200, + message: 'Outside diff', + rule_id: 'rule-b', + }, + ], + }, + }, + }); + + await service.syncRunningScanResults(); + + expect(mockGitHubAppService.createPullRequestReview).toHaveBeenCalledTimes(1); + expect(mockGitHubAppService.createPullRequestReview).toHaveBeenCalledWith( + 99999, + 'myorg', + 'myrepo', + 42, + expect.objectContaining({ + event: 'COMMENT', + comments: [ + expect.objectContaining({ + path: 'src/a.ts', + line: 2, + side: 'RIGHT', + }), + ], + }), + ); + expect(mockRepository.updateScanResult).toHaveBeenCalledWith('scan-1', { prReviewId: 900 }); + }); + + it('skips review posting when prReviewId already exists', async () => { + setupReviewContext(); + const scanResult = makeScanResult({ triggerRuleId: 'rule-1', prReviewId: 1234 }); + mockRepository.findRunningScanResults.mockResolvedValue([scanResult]); + mockTemporalService.describeWorkflow.mockResolvedValue({ + workflowId: 'shipsec-run-abc', + status: 'COMPLETED', + }); + mockTemporalService.getWorkflowResult.mockResolvedValue({ + outputs: { + normalize: { + summary: { critical: 0, high: 1, medium: 0, low: 0, info: 0 }, + findingCount: 1, + findings: [{ scanner: 'x', severity: 'high', file: 'src/a.ts', line: 2, message: 'x' }], + }, + }, + }); + + await service.syncRunningScanResults(); + + expect(mockGitHubAppService.getPullRequestFiles).not.toHaveBeenCalled(); + expect(mockGitHubAppService.createPullRequestReview).not.toHaveBeenCalled(); + }); + + it('falls back to body-only review on 422', async () => { + setupReviewContext(); + mockGitHubAppService.createPullRequestReview + .mockResolvedValueOnce({ status: 422, error: 'Validation failed' }) + .mockResolvedValueOnce({ + id: 901, + html_url: 'https://github.com/myorg/myrepo/pull/1#pullrequestreview-901', + }); + + const scanResult = makeScanResult({ triggerRuleId: 'rule-1' }); + mockRepository.findRunningScanResults.mockResolvedValue([scanResult]); + mockTemporalService.describeWorkflow.mockResolvedValue({ + workflowId: 'shipsec-run-abc', + status: 'COMPLETED', + }); + mockTemporalService.getWorkflowResult.mockResolvedValue({ + outputs: { + normalize: { + summary: { critical: 0, high: 1, medium: 0, low: 0, info: 0 }, + findingCount: 1, + findings: [{ scanner: 'x', severity: 'high', file: 'src/a.ts', line: 2, message: 'x' }], + }, + }, + }); + + await service.syncRunningScanResults(); + + expect(mockGitHubAppService.createPullRequestReview).toHaveBeenCalledTimes(2); + expect(mockGitHubAppService.createPullRequestReview.mock.calls[0][4].comments).toHaveLength( + 1, + ); + expect(mockGitHubAppService.createPullRequestReview.mock.calls[1][4].comments).toHaveLength( + 0, + ); + expect(mockRepository.updateScanResult).toHaveBeenCalledWith('scan-1', { prReviewId: 901 }); + }); + + it('logs warning and does not set prReviewId when fallback also returns 422', async () => { + setupReviewContext(); + mockGitHubAppService.createPullRequestReview + .mockResolvedValueOnce({ status: 422, error: 'Validation failed' }) + .mockResolvedValueOnce({ status: 422, error: 'Still invalid' }); + const warnSpy = vi.spyOn((service as any).logger, 'warn'); + + const scanResult = makeScanResult({ triggerRuleId: 'rule-1' }); + mockRepository.findRunningScanResults.mockResolvedValue([scanResult]); + mockTemporalService.describeWorkflow.mockResolvedValue({ + workflowId: 'shipsec-run-abc', + status: 'COMPLETED', + }); + mockTemporalService.getWorkflowResult.mockResolvedValue({ + outputs: { + normalize: { + summary: { critical: 0, high: 1, medium: 0, low: 0, info: 0 }, + findingCount: 1, + findings: [{ scanner: 'x', severity: 'high', file: 'src/a.ts', line: 2, message: 'x' }], + }, + }, + }); + + await service.syncRunningScanResults(); + + expect(mockGitHubAppService.createPullRequestReview).toHaveBeenCalledTimes(2); + expect( + mockRepository.updateScanResult.mock.calls.some( + (call) => call[0] === 'scan-1' && call[1]?.prReviewId != null, + ), + ).toBe(false); + expect(warnSpy).toHaveBeenCalled(); + }); + + it('includes capped finding count when inline-eligible findings exceed 50', async () => { + setupReviewContext(); + mockGitHubAppService.createPullRequestReview.mockResolvedValue({ + id: 902, + html_url: 'https://github.com/myorg/myrepo/pull/1#pullrequestreview-902', + }); + + const findings = Array.from({ length: 51 }, (_, i) => ({ + scanner: 'opengrep', + severity: 'high', + file: 'src/a.ts', + line: 2, + message: `issue-${i}`, + rule_id: `rule-${i}`, + })); + + const scanResult = makeScanResult({ triggerRuleId: 'rule-1' }); + mockRepository.findRunningScanResults.mockResolvedValue([scanResult]); + mockTemporalService.describeWorkflow.mockResolvedValue({ + workflowId: 'shipsec-run-abc', + status: 'COMPLETED', + }); + mockTemporalService.getWorkflowResult.mockResolvedValue({ + outputs: { + normalize: { + summary: { critical: 0, high: 51, medium: 0, low: 0, info: 0 }, + findingCount: 51, + findings, + }, + }, + }); + + await service.syncRunningScanResults(); + + const payload = mockGitHubAppService.createPullRequestReview.mock.calls[0][4]; + expect(payload.comments).toHaveLength(50); + expect(payload.body).toContain('additional inline-eligible findings not posted'); + }); + }); + + // ============ evaluateFailOn ============ + + describe('evaluateFailOn', () => { + it('should return success when failOn is none', () => { + const summary = { critical: 5, high: 3, medium: 2, low: 1, info: 10 }; + expect(service.evaluateFailOn('none', summary)).toBe('success'); + }); + + it('should return failure when failOn is info and any finding exists', () => { + expect( + service.evaluateFailOn('info', { critical: 0, high: 0, medium: 0, low: 0, info: 1 }), + ).toBe('failure'); + }); + + it('should return success when failOn is info and no findings exist', () => { + expect( + service.evaluateFailOn('info', { critical: 0, high: 0, medium: 0, low: 0, info: 0 }), + ).toBe('success'); + }); + + it('should return failure when failOn is critical and critical findings exist', () => { + expect( + service.evaluateFailOn('critical', { critical: 1, high: 0, medium: 0, low: 0, info: 0 }), + ).toBe('failure'); + }); + + it('should return success when failOn is critical and only high findings exist', () => { + expect( + service.evaluateFailOn('critical', { critical: 0, high: 5, medium: 0, low: 0, info: 0 }), + ).toBe('success'); + }); + + it('should return failure when failOn is high and critical findings exist', () => { + expect( + service.evaluateFailOn('high', { critical: 1, high: 0, medium: 0, low: 0, info: 0 }), + ).toBe('failure'); + }); + + it('should return failure when failOn is high and high findings exist', () => { + expect( + service.evaluateFailOn('high', { critical: 0, high: 1, medium: 0, low: 0, info: 0 }), + ).toBe('failure'); + }); + + it('should return success when failOn is high and only medium findings exist', () => { + expect( + service.evaluateFailOn('high', { critical: 0, high: 0, medium: 3, low: 0, info: 0 }), + ).toBe('success'); + }); + + it('should return failure when failOn is medium and medium or above exist', () => { + expect( + service.evaluateFailOn('medium', { critical: 0, high: 0, medium: 1, low: 0, info: 0 }), + ).toBe('failure'); + }); + + it('should return success when failOn is medium and only low findings exist', () => { + expect( + service.evaluateFailOn('medium', { critical: 0, high: 0, medium: 0, low: 5, info: 0 }), + ).toBe('success'); + }); + + it('should return success for unknown failOn value', () => { + expect( + service.evaluateFailOn('unknown', { critical: 5, high: 5, medium: 5, low: 5, info: 5 }), + ).toBe('success'); + }); + }); + + // ============ buildAnnotations ============ + + describe('buildAnnotations', () => { + it('should filter out findings without file', () => { + const findings = [ + { severity: 'high', line: 5, message: 'issue' }, + { file: 'src/a.ts', severity: 'high', line: 5, message: 'valid' }, + ]; + const annotations = service.buildAnnotations(findings); + expect(annotations).toHaveLength(1); + expect(annotations[0].path).toBe('src/a.ts'); + }); + + it('should filter out findings without valid line', () => { + const findings = [ + { file: 'a.ts', severity: 'high', line: 0, message: 'zero line' }, + { file: 'b.ts', severity: 'high', line: -1, message: 'negative line' }, + { file: 'c.ts', severity: 'high', line: 1.5, message: 'float line' }, + { file: 'd.ts', severity: 'high', message: 'no line' }, + { file: 'e.ts', severity: 'high', line: 10, message: 'valid' }, + ]; + const annotations = service.buildAnnotations(findings); + expect(annotations).toHaveLength(1); + expect(annotations[0].path).toBe('e.ts'); + }); + + it('should map severity to annotation level correctly', () => { + const findings = [ + { file: 'a.ts', severity: 'critical', line: 1, message: 'crit' }, + { file: 'b.ts', severity: 'high', line: 2, message: 'high' }, + { file: 'c.ts', severity: 'medium', line: 3, message: 'med' }, + { file: 'd.ts', severity: 'low', line: 4, message: 'low' }, + { file: 'e.ts', severity: 'info', line: 5, message: 'info' }, + ]; + const annotations = service.buildAnnotations(findings); + expect(annotations).toHaveLength(5); + expect(annotations[0].annotation_level).toBe('failure'); + expect(annotations[1].annotation_level).toBe('failure'); + expect(annotations[2].annotation_level).toBe('warning'); + expect(annotations[3].annotation_level).toBe('notice'); + expect(annotations[4].annotation_level).toBe('notice'); + }); + + it('should truncate to 50 annotations', () => { + const findings = Array.from({ length: 60 }, (_, i) => ({ + file: `file-${i}.ts`, + severity: 'high', + line: i + 1, + message: `finding ${i}`, + })); + const annotations = service.buildAnnotations(findings); + expect(annotations).toHaveLength(50); + }); + + it('should use endLine when valid', () => { + const findings = [ + { file: 'a.ts', severity: 'high', line: 5, endLine: 10, message: 'multi-line' }, + { file: 'b.ts', severity: 'high', line: 5, endLine: 3, message: 'invalid endLine' }, + ]; + const annotations = service.buildAnnotations(findings); + expect(annotations[0].start_line).toBe(5); + expect(annotations[0].end_line).toBe(10); + // Invalid endLine (< line) should fall back to line + expect(annotations[1].end_line).toBe(5); + }); + + it('should use type as title when available', () => { + const findings = [ + { file: 'a.ts', severity: 'high', line: 1, message: 'issue', type: 'trufflehog' }, + { file: 'b.ts', severity: 'high', line: 1, message: 'issue' }, + ]; + const annotations = service.buildAnnotations(findings); + expect(annotations[0].title).toBe('trufflehog'); + expect(annotations[1].title).toBeUndefined(); + }); + }); + + // ============ mapSeverityToAnnotationLevel ============ + + describe('mapSeverityToAnnotationLevel', () => { + it('should map critical to failure', () => { + expect(service.mapSeverityToAnnotationLevel('critical')).toBe('failure'); + }); + + it('should map high to failure', () => { + expect(service.mapSeverityToAnnotationLevel('high')).toBe('failure'); + }); + + it('should map medium to warning', () => { + expect(service.mapSeverityToAnnotationLevel('medium')).toBe('warning'); + }); + + it('should map low to notice', () => { + expect(service.mapSeverityToAnnotationLevel('low')).toBe('notice'); + }); + + it('should map info to notice', () => { + expect(service.mapSeverityToAnnotationLevel('info')).toBe('notice'); + }); + + it('should map unknown to notice', () => { + expect(service.mapSeverityToAnnotationLevel('unknown')).toBe('notice'); + }); + + it('should be case-insensitive', () => { + expect(service.mapSeverityToAnnotationLevel('CRITICAL')).toBe('failure'); + expect(service.mapSeverityToAnnotationLevel('HIGH')).toBe('failure'); + expect(service.mapSeverityToAnnotationLevel('Medium')).toBe('warning'); + }); + }); + + // ============ Check run finalization ============ + + describe('check run finalization', () => { + const setupCheckRunContext = () => { + mockRepository.findRepositoryById.mockResolvedValue({ + id: 'repo-1', + installationId: 'install-1', + fullName: 'myorg/myrepo', + }); + mockRepository.findInstallationById.mockResolvedValue({ + id: 'install-1', + installationId: 99999, + }); + }; + + it('should finalize check run on COMPLETED with findings', async () => { + setupCheckRunContext(); + mockRepository.findTriggerRuleById.mockResolvedValue({ + id: 'rule-1', + failOn: 'high', + }); + + const scanResult = makeScanResult({ + checkRunId: 5001, + triggerRuleId: 'rule-1', + }); + mockRepository.findRunningScanResults.mockResolvedValue([scanResult]); + mockTemporalService.describeWorkflow.mockResolvedValue({ + workflowId: 'shipsec-run-abc', + status: 'COMPLETED', + }); + mockTemporalService.getWorkflowResult.mockResolvedValue({ + outputs: { + 'normalize-1': { + summary: { critical: 0, high: 1, medium: 0, low: 0, info: 0 }, + findingCount: 1, + findings: [ + { + id: 'f-1', + scanner: 'trufflehog', + severity: 'high', + file: 'src/secrets.ts', + line: 10, + message: 'AWS key detected', + }, + ], + }, + }, + success: true, + }); + + await service.syncRunningScanResults(); + + // Check run should be finalized with failure (high >= high threshold) + expect(mockGitHubAppService.updateCheckRun).toHaveBeenCalledWith( + 99999, + 'myorg', + 'myrepo', + 5001, + expect.objectContaining({ + status: 'completed', + conclusion: 'failure', + output: expect.objectContaining({ + title: expect.stringContaining('failed'), + annotations: expect.arrayContaining([ + expect.objectContaining({ + path: 'src/secrets.ts', + start_line: 10, + annotation_level: 'failure', + message: 'AWS key detected', + }), + ]), + }), + }), + ); + + // Scan result should still be marked success (the workflow succeeded) + expect(mockRepository.updateScanResult).toHaveBeenCalledWith( + 'scan-1', + expect.objectContaining({ status: 'success' }), + ); + }); + + it('should finalize check run with success when below threshold', async () => { + setupCheckRunContext(); + mockRepository.findTriggerRuleById.mockResolvedValue({ + id: 'rule-1', + failOn: 'high', + }); + + const scanResult = makeScanResult({ + checkRunId: 5002, + triggerRuleId: 'rule-1', + }); + mockRepository.findRunningScanResults.mockResolvedValue([scanResult]); + mockTemporalService.describeWorkflow.mockResolvedValue({ + workflowId: 'shipsec-run-abc', + status: 'COMPLETED', + }); + mockTemporalService.getWorkflowResult.mockResolvedValue({ + outputs: { + 'normalize-1': { + summary: { critical: 0, high: 0, medium: 2, low: 1, info: 0 }, + findingCount: 3, + }, + }, + success: true, + }); + + await service.syncRunningScanResults(); + + // Check run should pass (only medium/low, threshold is high) + expect(mockGitHubAppService.updateCheckRun).toHaveBeenCalledWith( + 99999, + 'myorg', + 'myrepo', + 5002, + expect.objectContaining({ + status: 'completed', + conclusion: 'success', + output: expect.objectContaining({ + title: expect.stringContaining('passed'), + }), + }), + ); + }); + + it('should skip check run finalization when checkRunId is null', async () => { + const scanResult = makeScanResult({ checkRunId: null }); + mockRepository.findRunningScanResults.mockResolvedValue([scanResult]); + mockTemporalService.describeWorkflow.mockResolvedValue({ + workflowId: 'shipsec-run-abc', + status: 'COMPLETED', + }); + mockTemporalService.getWorkflowResult.mockResolvedValue({ + outputs: {}, + success: true, + }); + + await service.syncRunningScanResults(); + + expect(mockGitHubAppService.updateCheckRun).not.toHaveBeenCalled(); + expect(mockRepository.updateScanResult).toHaveBeenCalledWith( + 'scan-1', + expect.objectContaining({ status: 'success' }), + ); + }); + + it('should finalize check run with failure on FAILED workflow', async () => { + setupCheckRunContext(); + + const scanResult = makeScanResult({ checkRunId: 5003 }); + mockRepository.findRunningScanResults.mockResolvedValue([scanResult]); + mockTemporalService.describeWorkflow.mockResolvedValue({ + workflowId: 'shipsec-run-abc', + status: 'FAILED', + failure: { message: 'Component timeout' }, + }); + + await service.syncRunningScanResults(); + + expect(mockGitHubAppService.updateCheckRun).toHaveBeenCalledWith( + 99999, + 'myorg', + 'myrepo', + 5003, + expect.objectContaining({ + status: 'completed', + conclusion: 'failure', + }), + ); + + expect(mockRepository.updateScanResult).toHaveBeenCalledWith('scan-1', { + status: 'failure', + errorMessage: 'Component timeout', + completedAt: expect.any(Date), + }); + }); + + it('should finalize check run with cancelled on CANCELLED workflow', async () => { + setupCheckRunContext(); + + const scanResult = makeScanResult({ checkRunId: 5004 }); + mockRepository.findRunningScanResults.mockResolvedValue([scanResult]); + mockTemporalService.describeWorkflow.mockResolvedValue({ + workflowId: 'shipsec-run-abc', + status: 'CANCELLED', + failure: undefined, + }); + + await service.syncRunningScanResults(); + + expect(mockGitHubAppService.updateCheckRun).toHaveBeenCalledWith( + 99999, + 'myorg', + 'myrepo', + 5004, + expect.objectContaining({ + status: 'completed', + conclusion: 'cancelled', + }), + ); + }); + + it('should keep scan as running on transient check run update error', async () => { + setupCheckRunContext(); + + const scanResult = makeScanResult({ checkRunId: 5005 }); + mockRepository.findRunningScanResults.mockResolvedValue([scanResult]); + mockTemporalService.describeWorkflow.mockResolvedValue({ + workflowId: 'shipsec-run-abc', + status: 'COMPLETED', + }); + mockTemporalService.getWorkflowResult.mockResolvedValue({ + outputs: {}, + success: true, + }); + + mockGitHubAppService.updateCheckRun.mockRejectedValue(new Error('502 Bad Gateway')); + + await service.syncRunningScanResults(); + + // Scan should NOT be updated (kept as running for retry) + expect(mockRepository.updateScanResult).not.toHaveBeenCalled(); + }); + + it('should clear checkRunId and proceed when check run returns 404', async () => { + setupCheckRunContext(); + + const scanResult = makeScanResult({ checkRunId: 5006 }); + mockRepository.findRunningScanResults.mockResolvedValue([scanResult]); + mockTemporalService.describeWorkflow.mockResolvedValue({ + workflowId: 'shipsec-run-abc', + status: 'COMPLETED', + }); + mockTemporalService.getWorkflowResult.mockResolvedValue({ + outputs: {}, + success: true, + }); + + mockGitHubAppService.updateCheckRun.mockRejectedValue(new Error('404 Not Found')); + + await service.syncRunningScanResults(); + + // checkRunId should be cleared + expect(mockRepository.updateScanResult).toHaveBeenCalledWith('scan-1', { + checkRunId: null, + }); + // Scan should then be marked success (second call) + expect(mockRepository.updateScanResult).toHaveBeenCalledWith( + 'scan-1', + expect.objectContaining({ status: 'success' }), + ); + }); + + it('should clear checkRunId and proceed when check run returns 410', async () => { + setupCheckRunContext(); + + const scanResult = makeScanResult({ checkRunId: 5007 }); + mockRepository.findRunningScanResults.mockResolvedValue([scanResult]); + mockTemporalService.describeWorkflow.mockResolvedValue({ + workflowId: 'shipsec-run-abc', + status: 'COMPLETED', + }); + mockTemporalService.getWorkflowResult.mockResolvedValue({ + outputs: {}, + success: true, + }); + + mockGitHubAppService.updateCheckRun.mockRejectedValue(new Error('410 Gone')); + + await service.syncRunningScanResults(); + + // checkRunId should be cleared + expect(mockRepository.updateScanResult).toHaveBeenCalledWith('scan-1', { + checkRunId: null, + }); + }); + + it('should finalize check run on extraction failure with checkRunId', async () => { + setupCheckRunContext(); + + const scanResult = makeScanResult({ checkRunId: 5008 }); + mockRepository.findRunningScanResults.mockResolvedValue([scanResult]); + mockTemporalService.describeWorkflow.mockResolvedValue({ + workflowId: 'shipsec-run-abc', + status: 'COMPLETED', + }); + mockTemporalService.getWorkflowResult.mockRejectedValue(new Error('extraction error')); + + await service.syncRunningScanResults(); + + // Check run should be finalized with failure + expect(mockGitHubAppService.updateCheckRun).toHaveBeenCalledWith( + 99999, + 'myorg', + 'myrepo', + 5008, + expect.objectContaining({ + status: 'completed', + conclusion: 'failure', + }), + ); + + // Scan should be marked failure + expect(mockRepository.updateScanResult).toHaveBeenCalledWith('scan-1', { + status: 'failure', + errorMessage: expect.stringContaining('extraction error'), + completedAt: expect.any(Date), + }); + }); + }); +}); diff --git a/backend/src/github-app/diff-hunk-parser.ts b/backend/src/github-app/diff-hunk-parser.ts new file mode 100644 index 000000000..2264db052 --- /dev/null +++ b/backend/src/github-app/diff-hunk-parser.ts @@ -0,0 +1,81 @@ +export interface DiffHunk { + newStart: number; + newCount: number; +} + +const HUNK_HEADER_RE = /^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@/; +const WORKSPACE_PREFIXES = ['/workspace/repo/', '/scan/repo/', '/scan/', '/workspace/', '/repo/']; + +export function parseDiffHunks(patch: string): DiffHunk[] { + if (!patch) return []; + + const hunks: DiffHunk[] = []; + for (const line of patch.split('\n')) { + const match = HUNK_HEADER_RE.exec(line); + if (!match) continue; + + hunks.push({ + newStart: Number(match[1]), + newCount: match[2] == null ? 1 : Number(match[2]), + }); + } + + return hunks; +} + +export function isLineInDiffHunks(line: number, patch: string): boolean { + if (!Number.isInteger(line) || line < 1) return false; + + const hunks = parseDiffHunks(patch); + return hunks.some((hunk) => line >= hunk.newStart && line < hunk.newStart + hunk.newCount); +} + +export function buildPatchMap(files: { filename: string; patch?: string }[]): Map { + const map = new Map(); + + for (const file of files) { + if (!file.patch) continue; + map.set(file.filename, file.patch); + } + + return map; +} + +export function stripWorkspacePrefix(filePath: string): string { + for (const prefix of WORKSPACE_PREFIXES) { + if (filePath.startsWith(prefix)) { + return filePath.slice(prefix.length); + } + } + + return filePath.startsWith('/') ? filePath.slice(1) : filePath; +} + +export function escapeMarkdown(text: string): string { + return text.replace(/\|/g, '\\|').replace(//g, '>'); +} + +export function escapeBackticks(text: string): string { + return text.replace(/`/g, '``'); +} + +export function safeTruncate(text: string, max: number): string { + if (max < 1 || text.length <= max) return text; + + let truncated = text.slice(0, max); + const fenceCount = (truncated.match(/```/g) ?? []).length; + if (fenceCount % 2 === 1) { + const lastFence = truncated.lastIndexOf('```'); + if (lastFence >= 0) { + const newlineBeforeFence = truncated.lastIndexOf('\n', lastFence - 1); + truncated = newlineBeforeFence > 0 ? truncated.slice(0, newlineBeforeFence) : ''; + } else { + const lastNewline = truncated.lastIndexOf('\n'); + if (lastNewline > 0) { + truncated = truncated.slice(0, lastNewline); + } + } + } + + return `${truncated}\n...`; +} diff --git a/backend/src/github-app/dto/github-app.dto.ts b/backend/src/github-app/dto/github-app.dto.ts new file mode 100644 index 000000000..e5498dccd --- /dev/null +++ b/backend/src/github-app/dto/github-app.dto.ts @@ -0,0 +1,259 @@ +import { z } from 'zod'; + +// Installation DTOs +export const GitHubInstallationSchema = z.object({ + id: z.string().uuid(), + installationId: z.number(), + accountType: z.enum(['Organization', 'User']), + accountLogin: z.string(), + accountId: z.number(), + accountAvatarUrl: z.string().nullable(), + repositorySelection: z.enum(['all', 'selected']).nullable(), + isActive: z.boolean(), + organizationId: z.string(), + createdAt: z.string(), + updatedAt: z.string(), +}); + +export type GitHubInstallation = z.infer; + +// Repository DTOs +export const GitHubRepoSchema = z.object({ + id: z.string().uuid(), + installationId: z.string().uuid(), + repoId: z.number(), + fullName: z.string(), + name: z.string(), + owner: z.string(), + isPrivate: z.boolean(), + defaultBranch: z.string().nullable(), + description: z.string().nullable(), + language: z.string().nullable(), + htmlUrl: z.string().nullable(), + scansEnabled: z.boolean(), + lastSyncedAt: z.string().nullable(), + createdAt: z.string(), + updatedAt: z.string(), +}); + +export type GitHubRepo = z.infer; + +// Trigger Rule DTOs +export const CreateTriggerRuleSchema = z.object({ + name: z.string().min(1).max(191), + description: z.string().optional(), + repositoryPattern: z.string().min(1).max(255), + event: z.enum(['pull_request', 'push', 'release', 'repository_added']), + actions: z.array(z.string()).optional().default([]), + branches: z.array(z.string()).optional().default([]), + workflowId: z.string().uuid(), + postPrComment: z.boolean().optional().default(true), + createCheckRun: z.boolean().optional().default(true), + postPrReview: z.boolean().optional().default(false), + failOn: z.enum(['critical', 'high', 'medium', 'low', 'info', 'none']).optional().default('high'), + enabled: z.boolean().optional().default(true), + priority: z.number().optional().default(100), +}); + +export type CreateTriggerRuleInput = z.infer; + +export const UpdateTriggerRuleSchema = CreateTriggerRuleSchema.partial(); +export type UpdateTriggerRuleInput = z.infer; + +export const TriggerRuleSchema = z.object({ + id: z.string().uuid(), + name: z.string(), + description: z.string().nullable(), + repositoryPattern: z.string(), + event: z.string(), + actions: z.array(z.string()), + branches: z.array(z.string()), + workflowId: z.string().uuid(), + postPrComment: z.boolean(), + createCheckRun: z.boolean(), + postPrReview: z.boolean(), + failOn: z.string(), + enabled: z.boolean(), + priority: z.number(), + organizationId: z.string(), + createdBy: z.string().nullable(), + createdAt: z.string(), + updatedAt: z.string(), +}); + +export type TriggerRule = z.infer; + +// Scan Result DTOs +export const ScanSummarySchema = z.object({ + critical: z.number(), + high: z.number(), + medium: z.number(), + low: z.number(), + info: z.number(), +}); + +export type ScanSummary = z.infer; + +export const ScanFindingSchema = z.object({ + id: z.string(), + type: z.string(), + severity: z.enum(['critical', 'high', 'medium', 'low', 'info']), + file: z.string(), + line: z.number().optional(), + endLine: z.number().optional(), + message: z.string(), + snippet: z.string().optional(), + ruleId: z.string().optional(), + category: z.string().optional(), +}); + +export type ScanFinding = z.infer; + +export const GitHubScanResultSchema = z.object({ + id: z.string().uuid(), + repositoryId: z.string().uuid(), + workflowRunId: z.string(), + sourceType: z.enum(['pr', 'push', 'manual', 'schedule']), + triggerType: z.string().nullable(), + triggerSource: z.string().nullable(), + triggerLabel: z.string().nullable(), + scheduleId: z.string().nullable(), + scheduleName: z.string().nullable(), + prNumber: z.number().nullable(), + branch: z.string().nullable(), + commitSha: z.string().nullable(), + status: z.enum(['pending', 'running', 'success', 'failure', 'error']), + summary: ScanSummarySchema, + findingsCount: z.number(), + findings: z.array(ScanFindingSchema), + checkRunId: z.number().nullable(), + prCommentId: z.number().nullable(), + prReviewId: z.number().nullable(), + resultsUrl: z.string().nullable(), + errorMessage: z.string().nullable(), + triggerRuleId: z.string().uuid().nullable(), + organizationId: z.string(), + startedAt: z.string().nullable(), + completedAt: z.string().nullable(), + createdAt: z.string(), + updatedAt: z.string(), + workflowName: z.string().nullable(), + workflowVersion: z.number().nullable(), +}); + +export type GitHubScanResult = z.infer; + +// API Request DTOs +export const StartInstallationSchema = z.object({ + /** Optional: redirect URL after installation */ + redirectUri: z.string().url().optional(), +}); + +export type StartInstallationInput = z.infer; + +export const CompleteInstallationSchema = z.object({ + installationId: z.number(), + setupAction: z.enum(['install', 'update', 'request']).optional(), +}); + +export type CompleteInstallationInput = z.infer; + +export const TriggerScanSchema = z.object({ + workflowId: z.string().uuid(), + branch: z.string().optional(), + ref: z.string().optional(), + prNumber: z.number().optional(), +}); + +export type TriggerScanInput = z.infer; + +// GitHub Webhook Event Types +export interface GitHubWebhookEvent { + action?: string; + installation?: { + id: number; + account: { + login: string; + id: number; + type: string; + avatar_url?: string; + }; + repository_selection: 'all' | 'selected'; + permissions: Record; + }; + repositories?: { + id: number; + name: string; + full_name: string; + private: boolean; + }[]; + repositories_added?: { + id: number; + name: string; + full_name: string; + private: boolean; + }[]; + repositories_removed?: { + id: number; + name: string; + full_name: string; + }[]; + sender?: { + login: string; + id: number; + }; +} + +export interface GitHubPullRequestEvent { + action: string; + number: number; + pull_request: { + number: number; + title: string; + state: string; + head: { + ref: string; + sha: string; + }; + base: { + ref: string; + sha: string; + }; + user: { + login: string; + }; + }; + repository: { + id: number; + name: string; + full_name: string; + private: boolean; + default_branch: string; + clone_url: string; + html_url: string; + }; + installation?: { + id: number; + }; +} + +export interface GitHubPushEvent { + ref: string; + before: string; + after: string; + repository: { + id: number; + name: string; + full_name: string; + private: boolean; + default_branch: string; + clone_url: string; + html_url: string; + }; + pusher: { + name: string; + }; + installation?: { + id: number; + }; +} diff --git a/backend/src/github-app/github-app.controller.ts b/backend/src/github-app/github-app.controller.ts new file mode 100644 index 000000000..fe4c96509 --- /dev/null +++ b/backend/src/github-app/github-app.controller.ts @@ -0,0 +1,398 @@ +import { + Controller, + Get, + Post, + Patch, + Delete, + Body, + Param, + Query, + Headers, + Req, + HttpCode, + HttpStatus, + Logger, + BadRequestException, +} from '@nestjs/common'; +import type { Request } from 'express'; +import { createHmac, timingSafeEqual } from 'crypto'; +import { ConfigService } from '@nestjs/config'; +import { GitHubAppService } from './github-app.service'; +import { Public } from '../auth/public.decorator'; +import { CurrentAuth } from '../auth/auth-context.decorator'; +import type { AuthContext } from '../auth/types'; +import { + CreateTriggerRuleSchema, + UpdateTriggerRuleSchema, + type GitHubWebhookEvent, + type GitHubPullRequestEvent, + type GitHubPushEvent, +} from './dto/github-app.dto'; +import type { GitHubAppConfig } from '../config/github-app.config'; + +@Controller('github') +export class GitHubAppController { + private readonly logger = new Logger(GitHubAppController.name); + private readonly config: GitHubAppConfig; + + constructor( + private readonly githubAppService: GitHubAppService, + private readonly configService: ConfigService, + ) { + this.config = this.configService.get('githubApp')!; + } + + // ============ Configuration ============ + + @Get('status') + getStatus(@CurrentAuth() _auth: AuthContext) { + return { + configured: this.githubAppService.isConfigured(), + installUrl: this.githubAppService.isConfigured() + ? this.githubAppService.getInstallUrl() + : null, + }; + } + + // ============ Installations ============ + + @Get('installations') + async listInstallations(@CurrentAuth() auth: AuthContext) { + return this.githubAppService.listInstallations(auth); + } + + // ============ Repositories ============ + + @Get('repos') + async listRepositories(@CurrentAuth() auth: AuthContext) { + return this.githubAppService.listRepositories(auth); + } + + @Get('repos/:id') + async getRepository(@Param('id') id: string, @CurrentAuth() auth: AuthContext) { + return this.githubAppService.getRepository(id, auth); + } + + @Get('repos/:id/branches') + async listBranches(@Param('id') id: string, @CurrentAuth() auth: AuthContext) { + return this.githubAppService.listBranches(id, auth); + } + + @Patch('repos/:id/scans') + async toggleRepoScans( + @Param('id') id: string, + @Body() body: { enabled: boolean }, + @CurrentAuth() auth: AuthContext, + ) { + return this.githubAppService.toggleRepoScans(id, body.enabled, auth); + } + + /** + * Handle GitHub App post-installation setup callback. + * Called by the frontend after the user is redirected back from GitHub App installation. + * Ensures the installation record exists and repositories are synced. + */ + @Post('installations/setup') + async setupInstallation( + @Body() body: { installationId: number }, + @CurrentAuth() auth: AuthContext, + ) { + if (!body.installationId || typeof body.installationId !== 'number') { + throw new BadRequestException('installationId (number) is required'); + } + + return this.githubAppService.setupInstallation(body.installationId, auth); + } + + /** + * Get an installation token for worker components + * This endpoint is used by worker components to authenticate with GitHub + * when cloning private repositories or making API calls + */ + @Post('installations/:installationId/token') + async getInstallationToken( + @Param('installationId') installationId: string, + @CurrentAuth() auth: AuthContext, + ) { + const id = parseInt(installationId, 10); + if (isNaN(id)) { + throw new BadRequestException('Invalid installation ID'); + } + + const token = await this.githubAppService.getInstallationTokenForOrg(id, auth); + return { token }; + } + + @Post('repos/:id/scan') + async triggerScan( + @Param('id') id: string, + @Body() body: { workflowId?: string; branch?: string }, + @CurrentAuth() auth: AuthContext, + ) { + if (!body.workflowId) { + throw new BadRequestException('workflowId is required'); + } + + // Get repository to determine default branch if not provided + const repository = await this.githubAppService.getRepository(id, auth); + const branch = body.branch || repository.defaultBranch || undefined; + + const scanId = await this.githubAppService.triggerScan(id, body.workflowId, branch, auth); + return { scanId }; + } + + @Post('repos/sync') + async syncRepositories(@CurrentAuth() auth: AuthContext) { + // Get all installations for the organization + const installations = await this.githubAppService.listInstallations(auth); + + // Sync repositories for each installation + let totalSynced = 0; + for (const installation of installations) { + const count = await this.githubAppService.syncInstallationRepositories( + installation.id, + installation.installationId, + installation.organizationId, + { forceRefreshToken: true }, + ); + totalSynced += count; + } + + return { synced: totalSynced }; + } + + // ============ Trigger Rules ============ + + @Get('triggers') + async listTriggerRules(@CurrentAuth() auth: AuthContext) { + return this.githubAppService.listTriggerRules(auth); + } + + @Get('triggers/:id') + async getTriggerRule(@Param('id') id: string, @CurrentAuth() auth: AuthContext) { + return this.githubAppService.getTriggerRule(id, auth); + } + + @Post('triggers') + async createTriggerRule(@Body() body: unknown, @CurrentAuth() auth: AuthContext) { + const input = CreateTriggerRuleSchema.parse(body); + return this.githubAppService.createTriggerRule(input, auth); + } + + @Patch('triggers/:id') + async updateTriggerRule( + @Param('id') id: string, + @Body() body: unknown, + @CurrentAuth() auth: AuthContext, + ) { + const input = UpdateTriggerRuleSchema.parse(body); + return this.githubAppService.updateTriggerRule(id, input, auth); + } + + @Delete('triggers/:id') + @HttpCode(HttpStatus.NO_CONTENT) + async deleteTriggerRule(@Param('id') id: string, @CurrentAuth() auth: AuthContext) { + await this.githubAppService.deleteTriggerRule(id, auth); + } + + // ============ Scan Results ============ + + @Get('scans') + async listScanResults( + @CurrentAuth() auth: AuthContext, + @Query('repositoryId') repositoryId?: string, + @Query('status') status?: string, + @Query('source') source?: string, + @Query('scheduleId') scheduleId?: string, + @Query('triggerRuleId') triggerRuleId?: string, + @Query('limit') limit?: string, + @Query('offset') offset?: string, + ) { + const statuses = status + ?.split(',') + .map((value) => value.trim()) + .filter((value) => value.length > 0); + + const sourceValue = + source === 'pr' || source === 'push' || source === 'manual' || source === 'schedule' + ? source + : undefined; + + return this.githubAppService.listScanResults(auth, { + repositoryId, + statuses: statuses && statuses.length > 0 ? statuses : undefined, + source: sourceValue, + scheduleId: scheduleId || undefined, + triggerRuleId: triggerRuleId || undefined, + limit: limit ? parseInt(limit, 10) : undefined, + offset: offset ? parseInt(offset, 10) : undefined, + }); + } + + @Get('scans/:id') + async getScanResult(@Param('id') id: string, @CurrentAuth() auth: AuthContext) { + return this.githubAppService.getScanResult(id, auth); + } + + // ============ Webhook Handler ============ + + @Public() + @Post('webhook') + @HttpCode(HttpStatus.OK) + async handleWebhook( + @Headers('x-github-event') eventType: string, + @Headers('x-hub-signature-256') signature: string, + @Headers('x-github-delivery') deliveryId: string, + @Req() req: Request & { rawBody?: Buffer }, + ) { + this.logger.log(`Received GitHub webhook: ${eventType} (${deliveryId})`); + + // Verify signature + if (!this.verifyWebhookSignature(req.rawBody, signature)) { + this.logger.warn(`Invalid webhook signature for delivery ${deliveryId}`); + throw new BadRequestException('Invalid webhook signature'); + } + + const payload = JSON.parse(req.rawBody?.toString() ?? '{}'); + + // Resolve organization context from the installation record in our DB. + // For installation.created events the record does not exist yet — the + // authenticated POST /github/installations/setup endpoint handles that. + const githubInstallationId: number | undefined = payload.installation?.id; + let webhookAuth: AuthContext | null = null; + + if (githubInstallationId) { + webhookAuth = await this.githubAppService.resolveWebhookAuth(githubInstallationId); + } + + try { + switch (eventType) { + case 'installation': + return this.handleInstallationEvent(payload, webhookAuth); + + case 'installation_repositories': + if (!webhookAuth) { + this.logger.warn('installation_repositories webhook for unknown installation'); + return { status: 'unknown_installation' }; + } + return this.handleInstallationRepositoriesEvent(payload, webhookAuth); + + case 'pull_request': + if (!webhookAuth) { + this.logger.warn('pull_request webhook for unknown installation'); + return { status: 'unknown_installation' }; + } + return this.handlePullRequestEvent(payload as GitHubPullRequestEvent, webhookAuth); + + case 'push': + if (!webhookAuth) { + this.logger.warn('push webhook for unknown installation'); + return { status: 'unknown_installation' }; + } + return this.handlePushEvent(payload as GitHubPushEvent, webhookAuth); + + case 'ping': + this.logger.log('Received ping webhook'); + return { status: 'pong' }; + + default: + this.logger.log(`Unhandled event type: ${eventType}`); + return { status: 'ignored', event: eventType }; + } + } catch (error) { + this.logger.error(`Error handling webhook: ${error}`); + throw error; + } + } + + // ============ Webhook Event Handlers ============ + + private async handleInstallationEvent( + payload: GitHubWebhookEvent, + _webhookAuth: AuthContext | null, + ) { + const action = payload.action; + this.logger.log(`Installation event: ${action}`); + + switch (action) { + case 'created': + // Organization context is not available from the webhook payload. + // The authenticated POST /github/installations/setup endpoint + // (called by the frontend after the user is redirected back) will + // create the installation record with the correct organizationId. + this.logger.log( + `Installation created webhook received for ${payload.installation?.id}, awaiting setup callback`, + ); + return { status: 'pending_setup' }; + + case 'deleted': + await this.githubAppService.handleInstallationDeleted(payload); + return { status: 'deleted' }; + + case 'suspend': + await this.githubAppService.handleInstallationSuspended(payload); + return { status: 'suspended' }; + + case 'unsuspend': + await this.githubAppService.handleInstallationUnsuspended(payload); + return { status: 'unsuspended' }; + + default: + return { status: 'ignored', action }; + } + } + + private async handleInstallationRepositoriesEvent( + payload: GitHubWebhookEvent, + auth: AuthContext, + ) { + const action = payload.action; + this.logger.log(`Installation repositories event: ${action}`); + + switch (action) { + case 'added': + return this.githubAppService.handleRepositoriesAdded(payload, auth); + + case 'removed': + await this.githubAppService.handleRepositoriesRemoved(payload); + return { status: 'repositories_removed' }; + + default: + return { status: 'ignored', action }; + } + } + + private async handlePullRequestEvent(payload: GitHubPullRequestEvent, auth: AuthContext) { + return this.githubAppService.handlePullRequestEvent(payload, auth); + } + + private async handlePushEvent(payload: GitHubPushEvent, auth: AuthContext) { + return this.githubAppService.handlePushEvent(payload, auth); + } + + // ============ Signature Verification ============ + + private verifyWebhookSignature( + payload: Buffer | undefined, + signature: string | undefined, + ): boolean { + if (!this.config.webhookSecret) { + this.logger.warn('Webhook secret not configured, skipping signature verification'); + return true; + } + + if (!payload || !signature) { + return false; + } + + const expectedSignature = `sha256=${createHmac('sha256', this.config.webhookSecret) + .update(payload) + .digest('hex')}`; + + try { + return timingSafeEqual(Buffer.from(signature), Buffer.from(expectedSignature)); + } catch { + return false; + } + } +} diff --git a/backend/src/github-app/github-app.module.ts b/backend/src/github-app/github-app.module.ts new file mode 100644 index 000000000..aa71fca31 --- /dev/null +++ b/backend/src/github-app/github-app.module.ts @@ -0,0 +1,31 @@ +import { Global, Module, forwardRef } from '@nestjs/common'; +import { ConfigModule } from '@nestjs/config'; +import { GitHubAppController } from './github-app.controller'; +import { GitHubAppService } from './github-app.service'; +import { GitHubAppRepository } from './github-app.repository'; +import { ScanResultSyncService } from './scan-result-sync.service'; +import { githubAppConfig } from '../config/github-app.config'; +import { DatabaseModule } from '../database/database.module'; +import { WorkflowsModule } from '../workflows/workflows.module'; + +@Global() +@Module({ + imports: [ + DatabaseModule, + ConfigModule.forFeature(githubAppConfig), + forwardRef(() => WorkflowsModule), + ], + controllers: [GitHubAppController], + providers: [ + GitHubAppService, + GitHubAppRepository, + ScanResultSyncService, + { + provide: 'GitHubAppService', + useFactory: (service: GitHubAppService) => service, + inject: [GitHubAppService], + }, + ], + exports: [GitHubAppService, 'GitHubAppService'], +}) +export class GitHubAppModule {} diff --git a/backend/src/github-app/github-app.repository.ts b/backend/src/github-app/github-app.repository.ts new file mode 100644 index 000000000..7efd2c89b --- /dev/null +++ b/backend/src/github-app/github-app.repository.ts @@ -0,0 +1,469 @@ +import { Inject, Injectable } from '@nestjs/common'; +import { and, desc, eq, inArray, isNotNull, isNull, or, type SQL } from 'drizzle-orm'; +import type { NodePgDatabase } from 'drizzle-orm/node-postgres'; +import { DRIZZLE_TOKEN } from '../database/database.module'; + +import { + githubAppInstallations, + githubRepositories, + githubTriggerRules, + githubScanResults, + workflowRunsTable, + workflowsTable, + type GitHubAppInstallationRecord, + type NewGitHubAppInstallationRecord, + type GitHubRepositoryRecord, + type NewGitHubRepositoryRecord, + type GitHubTriggerRuleRecord, + type NewGitHubTriggerRuleRecord, + type GitHubScanResultRecord, + type NewGitHubScanResultRecord, +} from '../database/schema'; + +export type ScanResultWithWorkflow = GitHubScanResultRecord & { + workflowName: string | null; + workflowVersion: number | null; + triggerType: string | null; + triggerSource: string | null; + triggerLabel: string | null; +}; + +@Injectable() +export class GitHubAppRepository { + constructor( + @Inject(DRIZZLE_TOKEN) + private readonly db: NodePgDatabase, + ) {} + + // ============ Installation Methods ============ + + async createInstallation( + data: NewGitHubAppInstallationRecord, + ): Promise { + const [record] = await this.db.insert(githubAppInstallations).values(data).returning(); + return record; + } + + async findInstallationById(id: string): Promise { + const [record] = await this.db + .select() + .from(githubAppInstallations) + .where(eq(githubAppInstallations.id, id)) + .limit(1); + return record; + } + + async findInstallationByGitHubId( + installationId: number, + ): Promise { + const [record] = await this.db + .select() + .from(githubAppInstallations) + .where(eq(githubAppInstallations.installationId, installationId)) + .limit(1); + return record; + } + + async listInstallationsByOrg(organizationId: string): Promise { + return this.db + .select() + .from(githubAppInstallations) + .where( + and( + eq(githubAppInstallations.organizationId, organizationId), + eq(githubAppInstallations.isActive, true), + ), + ) + .orderBy(githubAppInstallations.createdAt); + } + + async updateInstallation( + id: string, + data: Partial, + ): Promise { + const [record] = await this.db + .update(githubAppInstallations) + .set({ ...data, updatedAt: new Date() }) + .where(eq(githubAppInstallations.id, id)) + .returning(); + return record; + } + + async deactivateInstallation(installationId: number): Promise { + await this.db + .update(githubAppInstallations) + .set({ isActive: false, updatedAt: new Date() }) + .where(eq(githubAppInstallations.installationId, installationId)); + } + + async deleteInstallation(id: string): Promise { + await this.db.delete(githubAppInstallations).where(eq(githubAppInstallations.id, id)); + } + + // ============ Repository Methods ============ + + async upsertRepository(data: NewGitHubRepositoryRecord): Promise { + const existing = await this.findRepositoryByGitHubId(data.repoId); + if (existing) { + const [updated] = await this.db + .update(githubRepositories) + .set({ ...data, updatedAt: new Date(), lastSyncedAt: new Date() }) + .where(eq(githubRepositories.id, existing.id)) + .returning(); + return updated; + } + const [record] = await this.db.insert(githubRepositories).values(data).returning(); + return record; + } + + async findRepositoryById(id: string): Promise { + const [record] = await this.db + .select() + .from(githubRepositories) + .where(eq(githubRepositories.id, id)) + .limit(1); + return record; + } + + async findRepositoryByGitHubId(repoId: number): Promise { + const [record] = await this.db + .select() + .from(githubRepositories) + .where(eq(githubRepositories.repoId, repoId)) + .limit(1); + return record; + } + + async findRepositoryByFullName(fullName: string): Promise { + const [record] = await this.db + .select() + .from(githubRepositories) + .where(eq(githubRepositories.fullName, fullName)) + .limit(1); + return record; + } + + async listRepositoriesByInstallation(installationId: string): Promise { + return this.db + .select() + .from(githubRepositories) + .where(eq(githubRepositories.installationId, installationId)) + .orderBy(githubRepositories.fullName); + } + + async listRepositoriesByOrg(organizationId: string): Promise { + return this.db + .select() + .from(githubRepositories) + .where(eq(githubRepositories.organizationId, organizationId)) + .orderBy(githubRepositories.fullName); + } + + async updateRepository( + id: string, + data: Partial, + ): Promise { + const [record] = await this.db + .update(githubRepositories) + .set({ ...data, updatedAt: new Date() }) + .where(eq(githubRepositories.id, id)) + .returning(); + return record; + } + + async deleteRepository(id: string): Promise { + await this.db.delete(githubRepositories).where(eq(githubRepositories.id, id)); + } + + async deleteRepositoriesByGitHubIds(repoIds: number[]): Promise { + if (repoIds.length === 0) return; + await this.db.delete(githubRepositories).where(inArray(githubRepositories.repoId, repoIds)); + } + + // ============ Trigger Rule Methods ============ + + async createTriggerRule(data: NewGitHubTriggerRuleRecord): Promise { + const [record] = await this.db.insert(githubTriggerRules).values(data).returning(); + return record; + } + + async findTriggerRuleById(id: string): Promise { + const [record] = await this.db + .select() + .from(githubTriggerRules) + .where(and(eq(githubTriggerRules.id, id), isNull(githubTriggerRules.deletedAt))) + .limit(1); + return record; + } + + async listTriggerRulesByOrg(organizationId: string): Promise { + return this.db + .select() + .from(githubTriggerRules) + .where( + and( + eq(githubTriggerRules.organizationId, organizationId), + isNull(githubTriggerRules.deletedAt), + ), + ) + .orderBy(githubTriggerRules.priority, githubTriggerRules.createdAt); + } + + async findMatchingTriggerRules( + organizationId: string, + event: string, + repoFullName: string, + ): Promise { + // Get all enabled rules for this org and event + const rules = await this.db + .select() + .from(githubTriggerRules) + .where( + and( + eq(githubTriggerRules.organizationId, organizationId), + eq(githubTriggerRules.event, event), + eq(githubTriggerRules.enabled, true), + isNull(githubTriggerRules.deletedAt), + ), + ) + .orderBy(githubTriggerRules.priority); + + // Filter by repository pattern + return rules.filter((rule) => this.matchesRepoPattern(rule.repositoryPattern, repoFullName)); + } + + private matchesRepoPattern(pattern: string, repoFullName: string): boolean { + if (pattern === '*') return true; + if (pattern.endsWith('/*')) { + const orgPrefix = pattern.slice(0, -2); + return repoFullName.startsWith(orgPrefix + '/'); + } + return pattern === repoFullName; + } + + async updateTriggerRule( + id: string, + data: Partial, + ): Promise { + const [record] = await this.db + .update(githubTriggerRules) + .set({ ...data, updatedAt: new Date() }) + .where(eq(githubTriggerRules.id, id)) + .returning(); + return record; + } + + async deleteTriggerRule(id: string): Promise { + await this.db + .update(githubTriggerRules) + .set({ + enabled: false, + deletedAt: new Date(), + updatedAt: new Date(), + }) + .where(eq(githubTriggerRules.id, id)); + } + + // ============ Scan Result Methods ============ + + async createScanResult(data: NewGitHubScanResultRecord): Promise { + const [record] = await this.db.insert(githubScanResults).values(data).returning(); + return record; + } + + async findScanResultById(id: string): Promise { + const [row] = await this.db + .select({ + scan: githubScanResults, + workflowName: workflowsTable.name, + workflowVersion: workflowRunsTable.workflowVersion, + triggerType: workflowRunsTable.triggerType, + triggerSource: workflowRunsTable.triggerSource, + triggerLabel: workflowRunsTable.triggerLabel, + }) + .from(githubScanResults) + .leftJoin(workflowRunsTable, eq(githubScanResults.workflowRunId, workflowRunsTable.runId)) + .leftJoin(workflowsTable, eq(workflowRunsTable.workflowId, workflowsTable.id)) + .where(eq(githubScanResults.id, id)) + .limit(1); + if (!row) return undefined; + return { + ...row.scan, + workflowName: row.workflowName, + workflowVersion: row.workflowVersion, + triggerType: row.triggerType ?? null, + triggerSource: row.triggerSource ?? null, + triggerLabel: row.triggerLabel ?? null, + }; + } + + async findScanResultByWorkflowRun( + workflowRunId: string, + ): Promise { + const [record] = await this.db + .select() + .from(githubScanResults) + .where(eq(githubScanResults.workflowRunId, workflowRunId)) + .limit(1); + return record; + } + + async listScanResultsByRepo( + repositoryId: string, + options?: { limit?: number; offset?: number }, + ): Promise { + let query = this.db + .select({ + scan: githubScanResults, + workflowName: workflowsTable.name, + workflowVersion: workflowRunsTable.workflowVersion, + triggerType: workflowRunsTable.triggerType, + triggerSource: workflowRunsTable.triggerSource, + triggerLabel: workflowRunsTable.triggerLabel, + }) + .from(githubScanResults) + .leftJoin(workflowRunsTable, eq(githubScanResults.workflowRunId, workflowRunsTable.runId)) + .leftJoin(workflowsTable, eq(workflowRunsTable.workflowId, workflowsTable.id)) + .where(eq(githubScanResults.repositoryId, repositoryId)) + .orderBy(desc(githubScanResults.createdAt)); + + if (options?.limit) { + query = query.limit(options.limit) as typeof query; + } + if (options?.offset) { + query = query.offset(options.offset) as typeof query; + } + + const rows = await query; + return rows.map((r) => ({ + ...r.scan, + workflowName: r.workflowName, + workflowVersion: r.workflowVersion, + triggerType: r.triggerType ?? null, + triggerSource: r.triggerSource ?? null, + triggerLabel: r.triggerLabel ?? null, + })); + } + + async listScanResultsByOrg( + organizationId: string, + options?: { + repositoryId?: string; + limit?: number; + offset?: number; + statuses?: string[]; + source?: 'pr' | 'push' | 'manual' | 'schedule'; + scheduleId?: string; + triggerRuleId?: string; + }, + ): Promise { + const conditions: SQL[] = [eq(githubScanResults.organizationId, organizationId)]; + + if (options?.repositoryId) { + conditions.push(eq(githubScanResults.repositoryId, options.repositoryId)); + } + + if (options?.statuses && options.statuses.length > 0) { + conditions.push(inArray(githubScanResults.status, options.statuses)); + } + + if (options?.source) { + if (options.source === 'schedule') { + conditions.push( + or( + eq(githubScanResults.sourceType, 'schedule'), + eq(workflowRunsTable.triggerType, 'schedule'), + )!, + ); + } else { + conditions.push(eq(githubScanResults.sourceType, options.source)); + } + } + + if (options?.scheduleId) { + conditions.push( + and( + eq(workflowRunsTable.triggerType, 'schedule'), + eq(workflowRunsTable.triggerSource, options.scheduleId), + )!, + ); + } + + if (options?.triggerRuleId) { + conditions.push(eq(githubScanResults.triggerRuleId, options.triggerRuleId)); + } + + let query = this.db + .select({ + scan: githubScanResults, + workflowName: workflowsTable.name, + workflowVersion: workflowRunsTable.workflowVersion, + triggerType: workflowRunsTable.triggerType, + triggerSource: workflowRunsTable.triggerSource, + triggerLabel: workflowRunsTable.triggerLabel, + }) + .from(githubScanResults) + .leftJoin(workflowRunsTable, eq(githubScanResults.workflowRunId, workflowRunsTable.runId)) + .leftJoin(workflowsTable, eq(workflowRunsTable.workflowId, workflowsTable.id)) + .where(and(...conditions)) + .orderBy(desc(githubScanResults.createdAt)); + + if (options?.limit) { + query = query.limit(options.limit) as typeof query; + } + if (options?.offset) { + query = query.offset(options.offset) as typeof query; + } + + const rows = await query; + return rows.map((r) => ({ + ...r.scan, + workflowName: r.workflowName, + workflowVersion: r.workflowVersion, + triggerType: r.triggerType ?? null, + triggerSource: r.triggerSource ?? null, + triggerLabel: r.triggerLabel ?? null, + })); + } + + async updateScanResult( + id: string, + data: Partial, + ): Promise { + const [record] = await this.db + .update(githubScanResults) + .set({ ...data, updatedAt: new Date() }) + .where(eq(githubScanResults.id, id)) + .returning(); + return record; + } + + async findLatestScanResultCommentForRule( + repositoryId: string, + prNumber: number, + triggerRuleId: string, + ): Promise { + const [record] = await this.db + .select() + .from(githubScanResults) + .where( + and( + eq(githubScanResults.repositoryId, repositoryId), + eq(githubScanResults.prNumber, prNumber), + eq(githubScanResults.triggerRuleId, triggerRuleId), + isNotNull(githubScanResults.prCommentId), + ), + ) + .orderBy(desc(githubScanResults.createdAt)) + .limit(1); + return record; + } + + async findRunningScanResults(limit = 50): Promise { + return this.db + .select() + .from(githubScanResults) + .where(eq(githubScanResults.status, 'running')) + .limit(limit); + } +} diff --git a/backend/src/github-app/github-app.service.ts b/backend/src/github-app/github-app.service.ts new file mode 100644 index 000000000..5bc604458 --- /dev/null +++ b/backend/src/github-app/github-app.service.ts @@ -0,0 +1,1533 @@ +import { + Injectable, + Logger, + BadRequestException, + NotFoundException, + OnModuleInit, + Inject, + forwardRef, +} from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import { createPrivateKey, createSign } from 'crypto'; +import type { AuthContext } from '../auth/types'; +import { GitHubAppRepository } from './github-app.repository'; +import { type GitHubAppConfig, isGitHubAppConfigured } from '../config/github-app.config'; +import type { + GitHubInstallation, + GitHubRepo, + TriggerRule, + GitHubScanResult, + CreateTriggerRuleInput, + UpdateTriggerRuleInput, + GitHubWebhookEvent, + GitHubPullRequestEvent, + GitHubPushEvent, +} from './dto/github-app.dto'; +import { WorkflowsService } from '../workflows/workflows.service'; + +interface InstallationTokenCache { + token: string; + expiresAt: Date; +} + +@Injectable() +export class GitHubAppService implements OnModuleInit { + private readonly logger = new Logger(GitHubAppService.name); + private config: GitHubAppConfig; + private installationTokenCache = new Map(); + + constructor( + private readonly configService: ConfigService, + private readonly repository: GitHubAppRepository, + @Inject(forwardRef(() => WorkflowsService)) + private readonly workflowsService: WorkflowsService, + ) { + this.config = this.configService.get('githubApp')!; + } + + async onModuleInit(): Promise { + if (isGitHubAppConfigured(this.config)) { + this.logger.log('GitHub App is configured and ready'); + } else { + this.logger.warn( + 'GitHub App is not fully configured. Set GITHUB_APP_ID and GITHUB_APP_PRIVATE_KEY.', + ); + } + } + + // ============ Webhook Auth Resolution ============ + + /** + * Resolve an AuthContext from a GitHub installation ID for webhook handling. + * Returns null if the installation is not found in our database (e.g. for + * brand-new installations that haven't been set up yet). + */ + async resolveWebhookAuth(githubInstallationId: number): Promise { + const installation = await this.repository.findInstallationByGitHubId(githubInstallationId); + if (!installation || !installation.organizationId) { + return null; + } + return { + userId: null, + organizationId: installation.organizationId, + roles: [], + isAuthenticated: false, + provider: 'github-webhook', + }; + } + + // ============ Configuration ============ + + isConfigured(): boolean { + return isGitHubAppConfigured(this.config); + } + + getInstallUrl(): string { + if (!this.config.appId) { + throw new BadRequestException('GitHub App is not configured'); + } + return `${this.config.baseUrl}/apps/shipsec-test-app/installations/new`; + } + + // ============ JWT Authentication ============ + + /** + * Generate a JWT for authenticating as the GitHub App + * JWTs are valid for 10 minutes + */ + private generateAppJwt(): string { + if (!this.config.appId || !this.config.privateKey) { + throw new BadRequestException('GitHub App is not configured'); + } + + const now = Math.floor(Date.now() / 1000); + const payload = { + iat: now - 60, // Issued 60 seconds ago to account for clock drift + exp: now + 600, // Expires in 10 minutes + iss: this.config.appId, + }; + + const header = { alg: 'RS256', typ: 'JWT' }; + const encodedHeader = this.base64UrlEncode(JSON.stringify(header)); + const encodedPayload = this.base64UrlEncode(JSON.stringify(payload)); + + const privateKey = createPrivateKey(this.config.privateKey); + const sign = createSign('RSA-SHA256'); + sign.update(`${encodedHeader}.${encodedPayload}`); + const signature = sign.sign(privateKey, 'base64url'); + + return `${encodedHeader}.${encodedPayload}.${signature}`; + } + + private base64UrlEncode(data: string): string { + return Buffer.from(data).toString('base64url'); + } + + // ============ Installation Token ============ + + /** + * Get an installation access token for making API calls on behalf of an installation + * Tokens are cached and automatically refreshed before expiry + */ + async getInstallationToken( + installationId: number, + options?: { forceRefresh?: boolean }, + ): Promise { + if (options?.forceRefresh) { + this.installationTokenCache.delete(installationId); + } + + const cached = this.installationTokenCache.get(installationId); + if (cached && cached.expiresAt > new Date(Date.now() + 60000)) { + return cached.token; + } + + const jwt = this.generateAppJwt(); + const response = await fetch( + `${this.config.apiBaseUrl}/app/installations/${installationId}/access_tokens`, + { + method: 'POST', + headers: { + Authorization: `Bearer ${jwt}`, + Accept: 'application/vnd.github+json', + 'X-GitHub-Api-Version': '2022-11-28', + }, + }, + ); + + if (!response.ok) { + const error = await response.text(); + this.logger.error(`Failed to get installation token: ${error}`); + throw new BadRequestException('Failed to authenticate with GitHub'); + } + + const data = (await response.json()) as { token: string; expires_at: string }; + const token = data.token; + const expiresAt = new Date(data.expires_at); + + this.installationTokenCache.set(installationId, { token, expiresAt }); + return token; + } + + /** + * Get an installation token after verifying the installation belongs to the + * caller's organization. Prevents cross-tenant token access. + */ + async getInstallationTokenForOrg(installationId: number, auth: AuthContext): Promise { + const organizationId = this.requireOrganizationId(auth); + const installation = await this.repository.findInstallationByGitHubId(installationId); + + if (!installation || installation.organizationId !== organizationId) { + throw new NotFoundException('Installation not found'); + } + + return this.getInstallationToken(installationId); + } + + /** + * Resolve an installation token from a repository full name (owner/repo). + * Looks up the repository record, finds the associated installation, + * and returns the token + GitHub installation ID. + */ + // ============ GitHub API Helpers ============ + + private async githubApiRequest( + installationId: number, + endpoint: string, + options?: RequestInit, + ): Promise { + const token = await this.getInstallationToken(installationId); + const url = endpoint.startsWith('http') ? endpoint : `${this.config.apiBaseUrl}${endpoint}`; + + const response = await fetch(url, { + ...options, + headers: { + Authorization: `token ${token}`, + Accept: 'application/vnd.github+json', + 'X-GitHub-Api-Version': '2022-11-28', + ...options?.headers, + }, + }); + + if (!response.ok) { + const error = await response.text(); + this.logger.error(`GitHub API error (${response.status}): ${error}`); + throw new BadRequestException(`GitHub API error: ${response.statusText}`); + } + + return response.json() as Promise; + } + + // ============ Installation Management ============ + + /** + * Setup a GitHub App installation after the user is redirected back from GitHub. + * If the installation already exists (webhook arrived first), returns it. + * Otherwise, fetches the installation details from GitHub API and creates it. + */ + async setupInstallation( + githubInstallationId: number, + auth: AuthContext, + ): Promise { + const organizationId = this.requireOrganizationId(auth); + + // Check if already exists (webhook may have arrived first) + const existing = await this.repository.findInstallationByGitHubId(githubInstallationId); + if (existing) { + this.logger.log( + `Installation ${githubInstallationId} already exists, ensuring repos are synced`, + ); + // Ensure repos are synced even if the installation exists + await this.syncInstallationRepositories( + existing.id, + githubInstallationId, + existing.organizationId, + ); + return this.mapInstallation(existing); + } + + // Fetch installation details from GitHub API using App JWT + const jwt = this.generateAppJwt(); + const response = await fetch( + `${this.config.apiBaseUrl}/app/installations/${githubInstallationId}`, + { + headers: { + Authorization: `Bearer ${jwt}`, + Accept: 'application/vnd.github+json', + 'X-GitHub-Api-Version': '2022-11-28', + }, + }, + ); + + if (!response.ok) { + const error = await response.text(); + this.logger.error( + `Failed to fetch installation ${githubInstallationId} from GitHub: ${error}`, + ); + throw new BadRequestException('Failed to fetch installation from GitHub'); + } + + const installationData = (await response.json()) as { + id: number; + account: { + type: string; + login: string; + id: number; + avatar_url: string; + }; + repository_selection: string; + permissions: Record; + }; + + // Create installation record + const record = await this.repository.createInstallation({ + installationId: githubInstallationId, + accountType: installationData.account.type, + accountLogin: installationData.account.login, + accountId: installationData.account.id, + accountAvatarUrl: installationData.account.avatar_url, + repositorySelection: installationData.repository_selection, + permissions: installationData.permissions, + organizationId, + installedBy: auth.userId, + isActive: true, + }); + + this.logger.log( + `Created installation ${githubInstallationId} for ${installationData.account.login} via setup callback`, + ); + + // Sync repositories + await this.syncInstallationRepositories(record.id, githubInstallationId, organizationId); + + return this.mapInstallation(record); + } + + async listInstallations(auth: AuthContext): Promise { + const organizationId = this.requireOrganizationId(auth); + const records = await this.repository.listInstallationsByOrg(organizationId); + + return records.map((r) => ({ + id: r.id, + installationId: r.installationId, + accountType: r.accountType as 'Organization' | 'User', + accountLogin: r.accountLogin, + accountId: r.accountId, + accountAvatarUrl: r.accountAvatarUrl, + repositorySelection: r.repositorySelection as 'all' | 'selected' | null, + isActive: r.isActive, + organizationId: r.organizationId, + createdAt: r.createdAt.toISOString(), + updatedAt: r.updatedAt.toISOString(), + })); + } + + async handleInstallationCreated( + event: GitHubWebhookEvent, + auth: AuthContext, + ): Promise { + const organizationId = this.requireOrganizationId(auth); + + if (!event.installation) { + throw new BadRequestException('Invalid installation event'); + } + + const { installation } = event; + + // Check if installation already exists + const existing = await this.repository.findInstallationByGitHubId(installation.id); + if (existing) { + // Update existing installation + const updated = await this.repository.updateInstallation(existing.id, { + accountType: installation.account.type, + accountLogin: installation.account.login, + accountId: installation.account.id, + accountAvatarUrl: installation.account.avatar_url, + repositorySelection: installation.repository_selection, + permissions: installation.permissions, + isActive: true, + suspendedAt: null, + }); + return this.mapInstallation(updated!); + } + + // Create new installation + const record = await this.repository.createInstallation({ + installationId: installation.id, + accountType: installation.account.type, + accountLogin: installation.account.login, + accountId: installation.account.id, + accountAvatarUrl: installation.account.avatar_url, + repositorySelection: installation.repository_selection, + permissions: installation.permissions, + organizationId, + installedBy: event.sender?.login ?? auth.userId, + isActive: true, + }); + + this.logger.log( + `Created GitHub installation ${installation.id} for ${installation.account.login}`, + ); + + // Sync repositories for this installation + await this.syncInstallationRepositories(record.id, installation.id, organizationId); + + return this.mapInstallation(record); + } + + async handleInstallationDeleted(event: GitHubWebhookEvent): Promise { + if (!event.installation) return; + + await this.repository.deactivateInstallation(event.installation.id); + this.installationTokenCache.delete(event.installation.id); + this.logger.log(`Deactivated GitHub installation ${event.installation.id}`); + } + + async handleInstallationSuspended(event: GitHubWebhookEvent): Promise { + if (!event.installation) return; + + const existing = await this.repository.findInstallationByGitHubId(event.installation.id); + if (existing) { + await this.repository.updateInstallation(existing.id, { + suspendedAt: new Date(), + }); + } + this.logger.log(`Suspended GitHub installation ${event.installation.id}`); + } + + async handleInstallationUnsuspended(event: GitHubWebhookEvent): Promise { + if (!event.installation) return; + + const existing = await this.repository.findInstallationByGitHubId(event.installation.id); + if (existing) { + await this.repository.updateInstallation(existing.id, { + suspendedAt: null, + isActive: true, + }); + } + this.logger.log(`Unsuspended GitHub installation ${event.installation.id}`); + } + + async handleRepositoriesAdded( + event: GitHubWebhookEvent, + auth: AuthContext, + ): Promise<{ status: string; scansTriggered?: number }> { + if (!event.installation || !event.repositories_added) { + return { status: 'ignored', scansTriggered: 0 }; + } + + // Installation tokens are scoped to selected repositories at creation time. + // Invalidate cache so newly granted repositories are visible immediately. + this.installationTokenCache.delete(event.installation.id); + + const organizationId = this.requireOrganizationId(auth); + const installation = await this.repository.findInstallationByGitHubId(event.installation.id); + if (!installation) { + return { status: 'installation_not_found', scansTriggered: 0 }; + } + + let scansTriggered = 0; + + for (const repo of event.repositories_added) { + const repoRecord = await this.repository.upsertRepository({ + installationId: installation.id, + repoId: repo.id, + fullName: repo.full_name, + name: repo.name, + owner: repo.full_name.split('/')[0], + isPrivate: repo.private, + organizationId, + }); + + const rules = await this.findMatchingTriggerRules( + organizationId, + 'repository_added', + repo.full_name, + ); + + for (const rule of rules) { + try { + const runHandle = await this.workflowsService.run( + rule.workflowId, + { + inputs: { + repository: repo.full_name, + owner: repo.full_name.split('/')[0], + repo: repo.full_name.split('/')[1], + installationId: event.installation.id, + repositoryId: repoRecord.id, + postPrComment: rule.postPrComment, + createCheckRun: rule.createCheckRun, + }, + }, + auth, + { + trigger: { + type: 'webhook', + sourceId: repo.full_name, + label: `Repository added: ${repo.full_name}`, + }, + componentInputs: this.buildGitHubComponentInputs(event.installation.id), + componentParams: { + 'github.repo.clone': { + repository: repo.full_name, + }, + }, + }, + ); + + await this.repository.createScanResult({ + repositoryId: repoRecord.id, + workflowRunId: runHandle.runId, + sourceType: 'manual', + status: 'running', + summary: { critical: 0, high: 0, medium: 0, low: 0, info: 0 }, + findingsCount: 0, + triggerRuleId: rule.id, + organizationId, + startedAt: new Date(), + }); + + scansTriggered++; + this.logger.log( + `Triggered scan for newly added repository ${repo.full_name} using rule ${rule.name} (workflow: ${runHandle.runId})`, + ); + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + this.logger.error(`Failed to trigger scan for rule ${rule.name}: ${errorMessage}`); + } + } + } + + this.logger.log(`Added ${event.repositories_added.length} repositories to installation`); + return { + status: scansTriggered > 0 ? 'scans_triggered' : 'repositories_added', + scansTriggered, + }; + } + + async handleRepositoriesRemoved(event: GitHubWebhookEvent): Promise { + if (!event.repositories_removed) return; + + if (event.installation) { + this.installationTokenCache.delete(event.installation.id); + } + + const repoIds = event.repositories_removed.map((r) => r.id); + await this.repository.deleteRepositoriesByGitHubIds(repoIds); + + this.logger.log(`Removed ${repoIds.length} repositories from installation`); + } + + // ============ Repository Management ============ + + async syncInstallationRepositories( + installationRecordId: string, + githubInstallationId: number, + organizationId: string, + options?: { forceRefreshToken?: boolean }, + ): Promise { + try { + if (options?.forceRefreshToken) { + this.installationTokenCache.delete(githubInstallationId); + } + + interface GitHubRepoResponse { + id: number; + name: string; + full_name: string; + private: boolean; + default_branch: string; + description: string | null; + language: string | null; + clone_url: string; + html_url: string; + owner: { login: string }; + } + + interface GitHubReposResponse { + total_count: number; + repositories: GitHubRepoResponse[]; + } + + let allRepos: GitHubRepoResponse[] = []; + let page = 1; + const perPage = 100; + let hasMore = true; + + // Paginate through all repositories + while (hasMore) { + const response = await this.githubApiRequest( + githubInstallationId, + `/installation/repositories?per_page=${perPage}&page=${page}`, + ); + + allRepos = allRepos.concat(response.repositories); + + // Check if there are more pages + hasMore = + response.repositories.length === perPage && allRepos.length < response.total_count; + page++; + } + + // Upsert all repositories with sync timestamp + const syncTimestamp = new Date(); + for (const repo of allRepos) { + await this.repository.upsertRepository({ + installationId: installationRecordId, + repoId: repo.id, + fullName: repo.full_name, + name: repo.name, + owner: repo.owner.login, + isPrivate: repo.private, + defaultBranch: repo.default_branch, + description: repo.description, + language: repo.language, + cloneUrl: repo.clone_url, + htmlUrl: repo.html_url, + organizationId, + lastSyncedAt: syncTimestamp, + }); + } + + this.logger.log( + `Synced ${allRepos.length} repositories for installation ${githubInstallationId}`, + ); + return allRepos.length; + } catch (error) { + this.logger.error(`Failed to sync repositories: ${error}`); + return 0; + } + } + + async listRepositories(auth: AuthContext): Promise { + const organizationId = this.requireOrganizationId(auth); + const records = await this.repository.listRepositoriesByOrg(organizationId); + + return records.map((r) => ({ + id: r.id, + installationId: r.installationId, + repoId: r.repoId, + fullName: r.fullName, + name: r.name, + owner: r.owner, + isPrivate: r.isPrivate, + defaultBranch: r.defaultBranch, + description: r.description, + language: r.language, + htmlUrl: r.htmlUrl, + scansEnabled: r.scansEnabled, + lastSyncedAt: r.lastSyncedAt?.toISOString() ?? null, + createdAt: r.createdAt.toISOString(), + updatedAt: r.updatedAt.toISOString(), + })); + } + + async getRepository(id: string, auth: AuthContext): Promise { + const organizationId = this.requireOrganizationId(auth); + const record = await this.repository.findRepositoryById(id); + + if (!record || record.organizationId !== organizationId) { + throw new NotFoundException(`Repository ${id} not found`); + } + + return { + id: record.id, + installationId: record.installationId, + repoId: record.repoId, + fullName: record.fullName, + name: record.name, + owner: record.owner, + isPrivate: record.isPrivate, + defaultBranch: record.defaultBranch, + description: record.description, + language: record.language, + htmlUrl: record.htmlUrl, + scansEnabled: record.scansEnabled, + lastSyncedAt: record.lastSyncedAt?.toISOString() ?? null, + createdAt: record.createdAt.toISOString(), + updatedAt: record.updatedAt.toISOString(), + }; + } + + async toggleRepoScans(id: string, enabled: boolean, auth: AuthContext): Promise { + const organizationId = this.requireOrganizationId(auth); + const record = await this.repository.findRepositoryById(id); + + if (!record || record.organizationId !== organizationId) { + throw new NotFoundException(`Repository ${id} not found`); + } + + const updated = await this.repository.updateRepository(id, { scansEnabled: enabled }); + return this.mapRepository(updated!); + } + + async listBranches( + id: string, + auth: AuthContext, + ): Promise<{ name: string; isDefault: boolean }[]> { + const organizationId = this.requireOrganizationId(auth); + const repo = await this.repository.findRepositoryById(id); + + if (!repo || repo.organizationId !== organizationId) { + throw new NotFoundException(`Repository ${id} not found`); + } + + const installation = await this.repository.findInstallationById(repo.installationId); + if (!installation) { + throw new BadRequestException('Repository installation not found'); + } + + const branches = await this.githubApiRequest<{ name: string }[]>( + installation.installationId, + `/repos/${repo.fullName}/branches?per_page=100`, + ); + + return branches.map((b) => ({ + name: b.name, + isDefault: b.name === repo.defaultBranch, + })); + } + + // ============ Trigger Rules ============ + + async listTriggerRules(auth: AuthContext): Promise { + const organizationId = this.requireOrganizationId(auth); + const records = await this.repository.listTriggerRulesByOrg(organizationId); + return records.map((r) => this.mapTriggerRule(r)); + } + + async getTriggerRule(id: string, auth: AuthContext): Promise { + const organizationId = this.requireOrganizationId(auth); + const record = await this.repository.findTriggerRuleById(id); + + if (!record || record.organizationId !== organizationId) { + throw new NotFoundException(`Trigger rule ${id} not found`); + } + + return this.mapTriggerRule(record); + } + + async createTriggerRule(input: CreateTriggerRuleInput, auth: AuthContext): Promise { + const organizationId = this.requireOrganizationId(auth); + + const record = await this.repository.createTriggerRule({ + name: input.name, + description: input.description ?? null, + repositoryPattern: input.repositoryPattern, + event: input.event, + actions: input.actions ?? [], + branches: input.branches ?? [], + workflowId: input.workflowId, + postPrComment: input.postPrComment ?? true, + createCheckRun: input.createCheckRun ?? true, + postPrReview: input.postPrReview ?? false, + enabled: input.enabled ?? true, + priority: input.priority ?? 100, + organizationId, + createdBy: auth.userId, + }); + + this.logger.log(`Created trigger rule ${record.id}: ${record.name}`); + return this.mapTriggerRule(record); + } + + async updateTriggerRule( + id: string, + input: UpdateTriggerRuleInput, + auth: AuthContext, + ): Promise { + const organizationId = this.requireOrganizationId(auth); + const existing = await this.repository.findTriggerRuleById(id); + + if (!existing || existing.organizationId !== organizationId) { + throw new NotFoundException(`Trigger rule ${id} not found`); + } + + const updated = await this.repository.updateTriggerRule(id, { + name: input.name, + description: input.description, + repositoryPattern: input.repositoryPattern, + event: input.event, + actions: input.actions, + branches: input.branches, + workflowId: input.workflowId, + postPrComment: input.postPrComment, + createCheckRun: input.createCheckRun, + postPrReview: input.postPrReview, + enabled: input.enabled, + priority: input.priority, + }); + + return this.mapTriggerRule(updated!); + } + + async deleteTriggerRule(id: string, auth: AuthContext): Promise { + const organizationId = this.requireOrganizationId(auth); + const existing = await this.repository.findTriggerRuleById(id); + + if (!existing || existing.organizationId !== organizationId) { + throw new NotFoundException(`Trigger rule ${id} not found`); + } + + await this.repository.deleteTriggerRule(id); + this.logger.log(`Soft-deleted trigger rule ${id}`); + } + + // ============ PR/Push Event Handling ============ + + async findMatchingTriggerRules( + organizationId: string, + event: string, + repoFullName: string, + action?: string, + branch?: string, + ): Promise { + const rules = await this.repository.findMatchingTriggerRules( + organizationId, + event, + repoFullName, + ); + + return rules + .filter((rule) => { + // Filter by action if specified + if (action && rule.actions && rule.actions.length > 0) { + if (!rule.actions.includes(action)) return false; + } + + // Filter by branch if specified + if (branch && rule.branches && rule.branches.length > 0) { + const branchMatches = rule.branches.some((pattern) => { + if (pattern.endsWith('/*')) { + return branch.startsWith(pattern.slice(0, -2)); + } + return pattern === branch; + }); + if (!branchMatches) return false; + } + + return true; + }) + .map((r) => this.mapTriggerRule(r)); + } + + /** + * Handle pull_request webhook events. + * Only processes: opened, synchronize, reopened actions. + * Creates scan results and triggers workflows for each matching rule. + */ + async handlePullRequestEvent( + event: GitHubPullRequestEvent, + auth: AuthContext, + ): Promise<{ status: string; scansTriggered?: number }> { + const { action, number, pull_request, repository, installation } = event; + const organizationId = this.requireOrganizationId(auth); + + this.logger.log(`PR event: ${action} on ${repository.full_name}#${number}`); + + // Only process specific actions + const processableActions = ['opened', 'synchronize', 'reopened']; + if (!processableActions.includes(action)) { + this.logger.log(`Ignoring PR action: ${action}`); + return { status: 'ignored', scansTriggered: 0 }; + } + + if (!installation) { + this.logger.warn('PR event without installation context'); + return { status: 'no_installation', scansTriggered: 0 }; + } + + // Find repository record by full_name + const repoRecord = await this.repository.findRepositoryByFullName(repository.full_name); + if (!repoRecord) { + this.logger.warn(`Repository ${repository.full_name} not found in database`); + return { status: 'repository_not_found', scansTriggered: 0 }; + } + + // Find matching trigger rules - use head.ref (source branch where changes are) + const rules = await this.findMatchingTriggerRules( + organizationId, + 'pull_request', + repository.full_name, + action, + pull_request.head.ref, + ); + + if (rules.length === 0) { + return { status: 'no_matching_rules', scansTriggered: 0 }; + } + + // Trigger workflows for each matching rule + const owner = repository.full_name.split('/')[0]; + let scansTriggered = 0; + for (const rule of rules) { + try { + // Build componentInputs with rule-scoped existingCommentId + const componentInputs = this.buildGitHubComponentInputs(installation.id); + + // Look up existing PR comment for this rule to enable upsert + if (rule.postPrComment) { + try { + const existingScan = await this.repository.findLatestScanResultCommentForRule( + repoRecord.id, + number, + rule.id, + ); + if (existingScan?.prCommentId) { + componentInputs['github.pr.comment'] = { + ...componentInputs['github.pr.comment'], + existingCommentId: existingScan.prCommentId, + }; + } + } catch (err) { + this.logger.warn( + `Failed to lookup existing comment for rule ${rule.name}: ${err instanceof Error ? err.message : err}`, + ); + } + } + + // Trigger workflow execution + const runHandle = await this.workflowsService.run( + rule.workflowId, + { + inputs: { + repository: repository.full_name, + owner, + repo: repository.name, + branch: pull_request.head.ref, + ref: pull_request.head.sha, + installationId: installation.id, + repositoryId: repoRecord.id, + prNumber: number, + postPrComment: rule.postPrComment, + createCheckRun: rule.createCheckRun, + postPrReview: rule.postPrReview, + }, + }, + auth, + { + trigger: { + type: 'webhook', + sourceId: `${repository.full_name}#${number}`, + label: `PR #${number}: ${pull_request.title}`, + }, + componentInputs, + componentParams: { + 'github.repo.clone': { + repository: repository.full_name, + branch: pull_request.head.ref, + ref: pull_request.head.sha, + }, + }, + }, + ); + + // Create scan result record first (always persisted) + const scanResult = await this.repository.createScanResult({ + repositoryId: repoRecord.id, + workflowRunId: runHandle.runId, + sourceType: 'pr', + prNumber: number, + branch: pull_request.head.ref, + commitSha: pull_request.head.sha, + status: 'running', + summary: { critical: 0, high: 0, medium: 0, low: 0, info: 0 }, + findingsCount: 0, + triggerRuleId: rule.id, + organizationId, + startedAt: new Date(), + }); + + // Best-effort: create check run immediately so it shows as in_progress + if (rule.createCheckRun) { + try { + const checkRun = await this.createCheckRun(installation.id, owner, repository.name, { + name: rule.name, + head_sha: pull_request.head.sha, + status: 'in_progress', + }); + await this.repository.updateScanResult(scanResult.id, { + checkRunId: checkRun.id, + }); + } catch (err) { + this.logger.warn( + `Failed to create check run for rule ${rule.name}: ${err instanceof Error ? err.message : err}`, + ); + } + } + + scansTriggered++; + this.logger.log( + `Triggered scan for PR #${number} using rule ${rule.name} (workflow: ${runHandle.runId})`, + ); + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + this.logger.error(`Failed to trigger scan for rule ${rule.name}: ${errorMessage}`); + } + } + + return { status: 'scans_triggered', scansTriggered }; + } + + /** + * Handle push webhook events. + * Ignores tag pushes (refs/tags/*). + * Creates scan results and triggers workflows for each matching rule. + */ + async handlePushEvent( + event: GitHubPushEvent, + auth: AuthContext, + ): Promise<{ status: string; scansTriggered?: number }> { + const { ref, after, repository, installation } = event; + const organizationId = this.requireOrganizationId(auth); + + // Parse branch name from ref + if (ref.startsWith('refs/tags/')) { + this.logger.log(`Ignoring tag push: ${ref}`); + return { status: 'ignored_tag', scansTriggered: 0 }; + } + + const branch = ref.replace('refs/heads/', ''); + this.logger.log(`Push event on ${repository.full_name}:${branch}`); + + if (!installation) { + this.logger.warn('Push event without installation context'); + return { status: 'no_installation', scansTriggered: 0 }; + } + + // Find repository record by full_name + const repoRecord = await this.repository.findRepositoryByFullName(repository.full_name); + if (!repoRecord) { + this.logger.warn(`Repository ${repository.full_name} not found in database`); + return { status: 'repository_not_found', scansTriggered: 0 }; + } + + // Find matching trigger rules + const rules = await this.findMatchingTriggerRules( + organizationId, + 'push', + repository.full_name, + undefined, + branch, + ); + + if (rules.length === 0) { + return { status: 'no_matching_rules', scansTriggered: 0 }; + } + + // Trigger workflows for each matching rule + const pushOwner = repository.full_name.split('/')[0]; + let scansTriggered = 0; + for (const rule of rules) { + try { + // Trigger workflow execution + const runHandle = await this.workflowsService.run( + rule.workflowId, + { + inputs: { + repository: repository.full_name, + owner: pushOwner, + repo: repository.name, + branch, + ref: after, + installationId: installation.id, + repositoryId: repoRecord.id, + postPrComment: rule.postPrComment, + createCheckRun: rule.createCheckRun, + }, + }, + auth, + { + trigger: { + type: 'webhook', + sourceId: `${repository.full_name}:${branch}`, + label: `Push to ${branch}`, + }, + componentInputs: this.buildGitHubComponentInputs(installation.id), + componentParams: { + 'github.repo.clone': { + repository: repository.full_name, + branch, + ref: after, + }, + }, + }, + ); + + // Create scan result record first (always persisted) + const scanResult = await this.repository.createScanResult({ + repositoryId: repoRecord.id, + workflowRunId: runHandle.runId, + sourceType: 'push', + branch, + commitSha: after, + status: 'running', + summary: { critical: 0, high: 0, medium: 0, low: 0, info: 0 }, + findingsCount: 0, + triggerRuleId: rule.id, + organizationId, + startedAt: new Date(), + }); + + // Best-effort: create check run immediately so it shows as in_progress + if (rule.createCheckRun) { + try { + const checkRun = await this.createCheckRun( + installation.id, + pushOwner, + repository.name, + { + name: rule.name, + head_sha: after, + status: 'in_progress', + }, + ); + await this.repository.updateScanResult(scanResult.id, { + checkRunId: checkRun.id, + }); + } catch (err) { + this.logger.warn( + `Failed to create check run for rule ${rule.name}: ${err instanceof Error ? err.message : err}`, + ); + } + } + + scansTriggered++; + this.logger.log( + `Triggered scan for push to ${branch} using rule ${rule.name} (workflow: ${runHandle.runId})`, + ); + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + this.logger.error(`Failed to trigger scan for rule ${rule.name}: ${errorMessage}`); + } + } + + return { status: 'scans_triggered', scansTriggered }; + } + + // ============ Scan Results ============ + + async listScanResults( + auth: AuthContext, + options?: { + repositoryId?: string; + limit?: number; + offset?: number; + statuses?: string[]; + source?: 'pr' | 'push' | 'manual' | 'schedule'; + scheduleId?: string; + triggerRuleId?: string; + }, + ): Promise { + const organizationId = this.requireOrganizationId(auth); + + const records = await this.repository.listScanResultsByOrg(organizationId, { + repositoryId: options?.repositoryId, + limit: options?.limit, + offset: options?.offset, + statuses: options?.statuses, + source: options?.source, + scheduleId: options?.scheduleId, + triggerRuleId: options?.triggerRuleId, + }); + + return records.map((r) => this.mapScanResult(r)); + } + + async getScanResult(id: string, auth: AuthContext): Promise { + const organizationId = this.requireOrganizationId(auth); + const record = await this.repository.findScanResultById(id); + + if (!record || record.organizationId !== organizationId) { + throw new NotFoundException(`Scan result ${id} not found`); + } + + return this.mapScanResult(record); + } + + /** + * Trigger a manual scan on a repository. + * Creates a scan result record and starts the workflow execution. + */ + async triggerScan( + repositoryId: string, + workflowId: string, + branch: string | undefined, + auth: AuthContext, + ): Promise { + const organizationId = this.requireOrganizationId(auth); + + // Fetch repository record and verify organization ownership + const repository = await this.repository.findRepositoryById(repositoryId); + if (!repository || repository.organizationId !== organizationId) { + throw new NotFoundException(`Repository ${repositoryId} not found`); + } + + // Get the installation record to get the GitHub installation ID + const installation = await this.repository.findInstallationById(repository.installationId); + if (!installation) { + throw new BadRequestException('Repository installation not found'); + } + + // Use provided branch or fall back to default branch + const targetBranch = branch || repository.defaultBranch || 'main'; + + try { + // Trigger workflow execution with GitHub repository inputs + // We do this first to get the runId before creating the scan result + const runHandle = await this.workflowsService.run( + workflowId, + { + inputs: { + repository: repository.fullName, + owner: repository.owner, + repo: repository.name, + branch: targetBranch, + installationId: installation.installationId, + repositoryId: repository.id, + }, + }, + auth, + { + trigger: { + type: 'manual', + sourceId: repository.fullName, + label: `Manual scan: ${repository.fullName} (${targetBranch})`, + }, + componentInputs: this.buildGitHubComponentInputs(installation.installationId), + componentParams: { + 'github.repo.clone': { + repository: repository.fullName, + branch: targetBranch, + }, + }, + }, + ); + + // Create scan result record with workflow run ID, status='running', source_type='manual' + const scanResult = await this.repository.createScanResult({ + repositoryId, + workflowRunId: runHandle.runId, + sourceType: 'manual', + branch: targetBranch, + status: 'running', + summary: { critical: 0, high: 0, medium: 0, low: 0, info: 0 }, + findingsCount: 0, + organizationId, + startedAt: new Date(), + }); + + this.logger.log( + `Started scan ${scanResult.id} for repository ${repository.fullName} (branch: ${targetBranch}, workflowRunId: ${runHandle.runId})`, + ); + + return scanResult.id; + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + this.logger.error(`Failed to start scan workflow: ${errorMessage}`); + throw error; + } + } + + // ============ GitHub API Operations ============ + + async postPrComment( + installationId: number, + owner: string, + repo: string, + prNumber: number, + body: string, + ): Promise<{ id: number }> { + return this.githubApiRequest<{ id: number }>( + installationId, + `/repos/${owner}/${repo}/issues/${prNumber}/comments`, + { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ body }), + }, + ); + } + + async getPullRequestFiles( + installationId: number, + owner: string, + repo: string, + prNumber: number, + ): Promise<{ filename: string; status: string; patch?: string }[]> { + const files: { filename: string; status: string; patch?: string }[] = []; + + for (let page = 1; page <= 30; page++) { + const pageFiles = await this.githubApiRequest< + { filename: string; status: string; patch?: string }[] + >( + installationId, + `/repos/${owner}/${repo}/pulls/${prNumber}/files?per_page=100&page=${page}`, + ); + + if (pageFiles.length === 0) break; + files.push(...pageFiles); + if (pageFiles.length < 100) break; + } + + return files; + } + + async createPullRequestReview( + installationId: number, + owner: string, + repo: string, + prNumber: number, + data: { + commit_id: string; + body: string; + event: 'COMMENT'; + comments: { path: string; line: number; side: 'RIGHT'; body: string }[]; + }, + ): Promise<{ id: number; html_url: string } | { status: 422; error: string }> { + const token = await this.getInstallationToken(installationId); + const url = `${this.config.apiBaseUrl}/repos/${owner}/${repo}/pulls/${prNumber}/reviews`; + const response = await fetch(url, { + method: 'POST', + headers: { + Authorization: `token ${token}`, + Accept: 'application/vnd.github+json', + 'Content-Type': 'application/json', + 'X-GitHub-Api-Version': '2022-11-28', + }, + body: JSON.stringify(data), + }); + + if (response.ok) { + return (await response.json()) as { id: number; html_url: string }; + } + + const error = await response.text(); + const excerpt = error.length > 500 ? `${error.slice(0, 500)}...` : error; + if (response.status === 422) { + return { status: 422, error: excerpt }; + } + + this.logger.error( + `GitHub review API error (${response.status}) for ${owner}/${repo}#${prNumber}: ${excerpt}`, + ); + throw new BadRequestException( + `GitHub review API error (${response.status}): ${response.statusText}`, + ); + } + + async createCheckRun( + installationId: number, + owner: string, + repo: string, + data: { + name: string; + head_sha: string; + status: 'queued' | 'in_progress' | 'completed'; + conclusion?: 'success' | 'failure' | 'neutral' | 'cancelled' | 'skipped'; + output?: { + title: string; + summary: string; + text?: string; + annotations?: { + path: string; + start_line: number; + end_line: number; + annotation_level: 'notice' | 'warning' | 'failure'; + message: string; + title?: string; + }[]; + }; + }, + ): Promise<{ id: number }> { + return this.githubApiRequest<{ id: number }>( + installationId, + `/repos/${owner}/${repo}/check-runs`, + { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(data), + }, + ); + } + + async updateCheckRun( + installationId: number, + owner: string, + repo: string, + checkRunId: number, + data: { + status?: 'queued' | 'in_progress' | 'completed'; + conclusion?: 'success' | 'failure' | 'neutral' | 'cancelled' | 'skipped'; + output?: { + title: string; + summary: string; + text?: string; + annotations?: { + path: string; + start_line: number; + end_line: number; + annotation_level: 'notice' | 'warning' | 'failure'; + message: string; + title?: string; + }[]; + }; + }, + ): Promise<{ id: number }> { + return this.githubApiRequest<{ id: number }>( + installationId, + `/repos/${owner}/${repo}/check-runs/${checkRunId}`, + { + method: 'PATCH', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(data), + }, + ); + } + + // ============ Helper Methods ============ + + /** + * Resolve the GitHub installation ID for a given repository full name. + * Used by WorkflowsService to auto-inject installationId for manual runs. + */ + async resolveInstallationForRepo( + repoFullName: string, + ): Promise<{ installationId: number } | null> { + const repo = await this.repository.findRepositoryByFullName(repoFullName); + if (!repo) { + return null; + } + + const installation = await this.repository.findInstallationById(repo.installationId); + if (!installation) { + return null; + } + + return { installationId: installation.installationId }; + } + + /** + * Build componentInputs map that injects installationId into all GitHub + * worker components. This ensures the token is resolved server-side at + * trigger time, not by the worker calling back to the backend. + */ + private buildGitHubComponentInputs( + installationId: number, + ): Record> { + const inputs = { installationId }; + return { + 'github.repo.clone': inputs, + 'github.pr.context': inputs, + 'github.pr.comment': inputs, + 'github.check.create': inputs, + 'github.check.update': inputs, + }; + } + + private requireOrganizationId(auth: AuthContext | null): string { + if (!auth?.organizationId) { + throw new BadRequestException('Organization context is required'); + } + return auth.organizationId; + } + + private mapInstallation(record: any): GitHubInstallation { + return { + id: record.id, + installationId: record.installationId, + accountType: record.accountType, + accountLogin: record.accountLogin, + accountId: record.accountId, + accountAvatarUrl: record.accountAvatarUrl, + repositorySelection: record.repositorySelection, + isActive: record.isActive, + organizationId: record.organizationId, + createdAt: record.createdAt.toISOString(), + updatedAt: record.updatedAt.toISOString(), + }; + } + + private mapRepository(record: any): GitHubRepo { + return { + id: record.id, + installationId: record.installationId, + repoId: record.repoId, + fullName: record.fullName, + name: record.name, + owner: record.owner, + isPrivate: record.isPrivate, + defaultBranch: record.defaultBranch, + description: record.description, + language: record.language, + htmlUrl: record.htmlUrl, + scansEnabled: record.scansEnabled, + lastSyncedAt: record.lastSyncedAt?.toISOString() ?? null, + createdAt: record.createdAt.toISOString(), + updatedAt: record.updatedAt.toISOString(), + }; + } + + private mapTriggerRule(record: any): TriggerRule { + return { + id: record.id, + name: record.name, + description: record.description, + repositoryPattern: record.repositoryPattern, + event: record.event, + actions: record.actions ?? [], + branches: record.branches ?? [], + workflowId: record.workflowId, + postPrComment: record.postPrComment, + createCheckRun: record.createCheckRun, + postPrReview: record.postPrReview, + failOn: record.failOn, + enabled: record.enabled, + priority: record.priority, + organizationId: record.organizationId, + createdBy: record.createdBy, + createdAt: record.createdAt.toISOString(), + updatedAt: record.updatedAt.toISOString(), + }; + } + + private mapScanResult(record: any): GitHubScanResult { + const triggerType = typeof record.triggerType === 'string' ? record.triggerType : null; + const triggerSource = typeof record.triggerSource === 'string' ? record.triggerSource : null; + const triggerLabel = typeof record.triggerLabel === 'string' ? record.triggerLabel : null; + const isScheduleTriggered = triggerType === 'schedule'; + + return { + id: record.id, + repositoryId: record.repositoryId, + workflowRunId: record.workflowRunId, + sourceType: isScheduleTriggered ? 'schedule' : record.sourceType, + triggerType, + triggerSource, + triggerLabel, + scheduleId: isScheduleTriggered ? triggerSource : null, + scheduleName: isScheduleTriggered ? triggerLabel : null, + prNumber: record.prNumber, + branch: record.branch, + commitSha: record.commitSha, + status: record.status, + summary: record.summary ?? { critical: 0, high: 0, medium: 0, low: 0, info: 0 }, + findingsCount: record.findingsCount, + findings: record.findings ?? [], + checkRunId: record.checkRunId, + prCommentId: record.prCommentId, + prReviewId: record.prReviewId ?? null, + resultsUrl: record.resultsUrl, + errorMessage: record.errorMessage, + triggerRuleId: record.triggerRuleId, + organizationId: record.organizationId, + startedAt: record.startedAt?.toISOString() ?? null, + completedAt: record.completedAt?.toISOString() ?? null, + createdAt: record.createdAt.toISOString(), + updatedAt: record.updatedAt.toISOString(), + workflowName: record.workflowName ?? null, + workflowVersion: record.workflowVersion ?? null, + }; + } +} diff --git a/backend/src/github-app/index.ts b/backend/src/github-app/index.ts new file mode 100644 index 000000000..7271a1143 --- /dev/null +++ b/backend/src/github-app/index.ts @@ -0,0 +1,4 @@ +export * from './github-app.module'; +export * from './github-app.service'; +export * from './github-app.repository'; +export * from './dto/github-app.dto'; diff --git a/backend/src/github-app/scan-result-sync.service.ts b/backend/src/github-app/scan-result-sync.service.ts new file mode 100644 index 000000000..0e7ee1d72 --- /dev/null +++ b/backend/src/github-app/scan-result-sync.service.ts @@ -0,0 +1,787 @@ +import { Injectable, Logger, OnModuleInit, OnModuleDestroy } from '@nestjs/common'; +import { GitHubAppRepository } from './github-app.repository'; +import { GitHubAppService } from './github-app.service'; +import { TemporalService } from '../temporal/temporal.service'; +import type { GitHubScanResultRecord, NewGitHubScanResultRecord } from '../database/schema'; +import { + buildPatchMap, + escapeBackticks, + escapeMarkdown, + isLineInDiffHunks, + safeTruncate, + stripWorkspacePrefix, +} from './diff-hunk-parser'; + +const TERMINAL_STATUSES = ['COMPLETED', 'FAILED', 'CANCELLED', 'TERMINATED', 'TIMED_OUT']; +const POLL_INTERVAL_MS = 30_000; // 30 seconds +const MAX_ANNOTATIONS = 50; +const MAX_REVIEW_INLINE_COMMENTS = 50; +const MAX_REVIEW_SKIPPED_IN_BODY = 20; +const MAX_REVIEW_BODY_LENGTH = 65_535; +const MAX_REVIEW_COMMENT_LENGTH = 65_535; +const MAX_REVIEW_SNIPPET_LENGTH = 500; + +/** Severity levels in descending order of severity */ +const SEVERITY_ORDER = ['critical', 'high', 'medium', 'low', 'info'] as const; + +interface ScanSummary { + critical: number; + high: number; + medium: number; + low: number; + info: number; +} + +interface ReviewFindingSummary { + file: string; + line: number; + severity: 'critical' | 'high' | 'medium' | 'low' | 'info'; + message: string; + ruleId?: string; + category?: string; + snippet?: string; +} + +/** + * Polls for scan results in 'running' state and syncs their status + * with the corresponding Temporal workflow execution. + * Also finalizes GitHub check runs with conclusions, annotations, and output. + */ +@Injectable() +export class ScanResultSyncService implements OnModuleInit, OnModuleDestroy { + private readonly logger = new Logger(ScanResultSyncService.name); + private intervalHandle: ReturnType | null = null; + private syncing = false; + + constructor( + private readonly repository: GitHubAppRepository, + private readonly temporalService: TemporalService, + private readonly githubAppService: GitHubAppService, + ) {} + + onModuleInit() { + this.intervalHandle = setInterval(() => { + this.syncRunningScanResults().catch((err) => { + this.logger.error(`Scan result sync failed: ${err}`); + }); + }, POLL_INTERVAL_MS); + this.logger.log('Scan result sync service started'); + } + + onModuleDestroy() { + if (this.intervalHandle) { + clearInterval(this.intervalHandle); + this.intervalHandle = null; + } + } + + async syncRunningScanResults(): Promise { + // Guard against concurrent runs + if (this.syncing) return; + this.syncing = true; + + try { + const runningScanResults = await this.repository.findRunningScanResults(); + if (runningScanResults.length === 0) return; + + this.logger.debug(`Syncing ${runningScanResults.length} running scan results`); + + for (const scanResult of runningScanResults) { + try { + await this.syncScanResult(scanResult); + } catch (err) { + this.logger.warn( + `Failed to sync scan result ${scanResult.id}: ${err instanceof Error ? err.message : err}`, + ); + } + } + } finally { + this.syncing = false; + } + } + + private async syncScanResult(scanResult: GitHubScanResultRecord): Promise { + let temporalStatus; + try { + temporalStatus = await this.temporalService.describeWorkflow({ + workflowId: scanResult.workflowRunId, + }); + } catch { + // Workflow not found in Temporal — mark scan as failed + this.logger.warn( + `Workflow run ${scanResult.workflowRunId} not found in Temporal, marking scan as failed`, + ); + await this.finalizeCheckRun(scanResult, 'failure', 'Workflow run not found'); + await this.repository.updateScanResult(scanResult.id, { + status: 'failure', + errorMessage: 'Workflow run not found', + completedAt: new Date(), + }); + return; + } + + if (!TERMINAL_STATUSES.includes(temporalStatus.status)) { + return; // Still running + } + + if (temporalStatus.status === 'COMPLETED') { + let extractedData: Partial; + try { + const result = await this.temporalService.getWorkflowResult({ + workflowId: scanResult.workflowRunId, + }); + extractedData = this.extractScanDataFromResult(result); + } catch (err) { + // Extraction failure after COMPLETED — mark as failure per spec + const extractionError = `Failed to extract results: ${err instanceof Error ? err.message : err}`; + this.logger.warn(`Could not extract results for scan ${scanResult.id}: ${extractionError}`); + await this.finalizeCheckRun(scanResult, 'failure', extractionError); + await this.repository.updateScanResult(scanResult.id, { + status: 'failure', + errorMessage: extractionError, + completedAt: new Date(), + }); + return; + } + + // Determine check run conclusion based on failOn threshold + const summary = (extractedData.summary ?? scanResult.summary) as ScanSummary; + const conclusion = await this.determineConclusion(scanResult, summary); + + // Finalize check run before persisting terminal scan status + const annotations = this.buildAnnotations(extractedData.findings ?? []); + const finalized = await this.finalizeCheckRunWithFindings( + scanResult, + conclusion, + summary, + annotations, + ); + + if (!finalized) { + // Transient error — keep scan as running, retry next poll + return; + } + + // Best-effort PR review posting, controlled by trigger rule. + try { + await this.postPrReviewIfEnabled(scanResult, extractedData.findings ?? []); + } catch (err) { + this.logger.warn(`Failed to post PR review for scan ${scanResult.id}: ${err}`); + } + + await this.repository.updateScanResult(scanResult.id, { + status: 'success', + ...extractedData, + completedAt: new Date(), + }); + + this.logger.log( + `Scan result ${scanResult.id} completed: ${extractedData.findingsCount ?? 0} findings`, + ); + } else { + // FAILED, CANCELLED, TERMINATED, TIMED_OUT + const failureReason = + temporalStatus.failure && typeof temporalStatus.failure === 'object' + ? ((temporalStatus.failure as Record).message ?? + (temporalStatus.failure as Record).reason) + : undefined; + + const errorMessage = + typeof failureReason === 'string' ? failureReason : `Workflow ${temporalStatus.status}`; + + const checkConclusion = temporalStatus.status === 'CANCELLED' ? 'cancelled' : 'failure'; + await this.finalizeCheckRun(scanResult, checkConclusion, errorMessage); + + await this.repository.updateScanResult(scanResult.id, { + status: 'failure', + errorMessage, + completedAt: new Date(), + }); + + this.logger.log(`Scan result ${scanResult.id} failed: ${temporalStatus.status}`); + } + } + + /** + * Determine the check run conclusion by evaluating the failOn threshold + * from the trigger rule against the scan summary. + */ + private async determineConclusion( + scanResult: GitHubScanResultRecord, + summary: ScanSummary, + ): Promise<'success' | 'failure'> { + if (!scanResult.triggerRuleId) return 'success'; + + try { + const rule = await this.repository.findTriggerRuleById(scanResult.triggerRuleId); + if (!rule) return 'success'; + return this.evaluateFailOn(rule.failOn, summary); + } catch { + return 'success'; // If we can't load the rule, don't block + } + } + + /** + * Evaluate whether the check run should fail based on the failOn threshold. + */ + evaluateFailOn(failOn: string, summary: ScanSummary): 'success' | 'failure' { + if (failOn === 'none') return 'success'; + + const thresholdIndex = SEVERITY_ORDER.indexOf(failOn as (typeof SEVERITY_ORDER)[number]); + if (thresholdIndex === -1) return 'success'; + + // Check all severities at or above the threshold + for (let i = 0; i <= thresholdIndex; i++) { + const level = SEVERITY_ORDER[i]; + if (summary[level] > 0) return 'failure'; + } + + return 'success'; + } + + /** + * Build GitHub annotations from findings with valid file + positive line. + * Truncates to MAX_ANNOTATIONS (50). + */ + buildAnnotations(findings: Record[]): { + path: string; + start_line: number; + end_line: number; + annotation_level: 'notice' | 'warning' | 'failure'; + message: string; + title?: string; + }[] { + const annotations: { + path: string; + start_line: number; + end_line: number; + annotation_level: 'notice' | 'warning' | 'failure'; + message: string; + title?: string; + }[] = []; + + for (const finding of findings) { + if (annotations.length >= MAX_ANNOTATIONS) break; + + const file = finding.file; + const line = finding.line; + + // Only include findings with valid file + positive integer line + if (typeof file !== 'string' || !file) continue; + if (typeof line !== 'number' || line < 1 || !Number.isInteger(line)) continue; + + const endLine = + typeof finding.endLine === 'number' && finding.endLine >= line ? finding.endLine : line; + + annotations.push({ + path: file, + start_line: line, + end_line: endLine, + annotation_level: this.mapSeverityToAnnotationLevel( + typeof finding.severity === 'string' ? finding.severity : 'info', + ), + message: typeof finding.message === 'string' ? finding.message : 'Finding detected', + title: typeof finding.type === 'string' ? finding.type : undefined, + }); + } + + return annotations; + } + + /** + * Map severity to GitHub annotation level. + */ + mapSeverityToAnnotationLevel(severity: string): 'notice' | 'warning' | 'failure' { + switch (severity.toLowerCase()) { + case 'critical': + case 'high': + return 'failure'; + case 'medium': + return 'warning'; + default: + return 'notice'; + } + } + + /** + * Simple check run finalization (no findings/annotations). + * Used for non-completed workflows (failed, cancelled, etc.). + */ + private async finalizeCheckRun( + scanResult: GitHubScanResultRecord, + conclusion: 'success' | 'failure' | 'cancelled', + message: string, + ): Promise { + if (!scanResult.checkRunId) return true; + + try { + const { installationId, owner, repo } = await this.resolveCheckRunContext(scanResult); + + await this.githubAppService.updateCheckRun( + installationId, + owner, + repo, + scanResult.checkRunId, + { + status: 'completed', + conclusion, + output: { + title: conclusion === 'success' ? 'Scan passed' : 'Scan failed', + summary: message, + }, + }, + ); + return true; + } catch (err) { + return this.handleCheckRunError(err, scanResult); + } + } + + /** + * Finalize check run with findings data, summary, and annotations. + */ + private async finalizeCheckRunWithFindings( + scanResult: GitHubScanResultRecord, + conclusion: 'success' | 'failure', + summary: ScanSummary, + annotations: { + path: string; + start_line: number; + end_line: number; + annotation_level: 'notice' | 'warning' | 'failure'; + message: string; + title?: string; + }[], + ): Promise { + if (!scanResult.checkRunId) return true; + + try { + const { installationId, owner, repo } = await this.resolveCheckRunContext(scanResult); + + const totalFindings = + summary.critical + summary.high + summary.medium + summary.low + summary.info; + const title = + conclusion === 'success' + ? totalFindings > 0 + ? `Scan passed — ${totalFindings} finding(s)` + : 'Scan passed — no findings' + : `Scan failed — ${totalFindings} finding(s)`; + + const summaryLines = [ + `| Severity | Count |`, + `|----------|-------|`, + `| Critical | ${summary.critical} |`, + `| High | ${summary.high} |`, + `| Medium | ${summary.medium} |`, + `| Low | ${summary.low} |`, + `| Info | ${summary.info} |`, + ]; + + if (annotations.length < totalFindings) { + summaryLines.push( + '', + `> Showing ${annotations.length} of ${totalFindings} findings as annotations.`, + ); + } + + await this.githubAppService.updateCheckRun( + installationId, + owner, + repo, + scanResult.checkRunId, + { + status: 'completed', + conclusion, + output: { + title, + summary: summaryLines.join('\n'), + annotations: annotations.length > 0 ? annotations : undefined, + }, + }, + ); + return true; + } catch (err) { + return this.handleCheckRunError(err, scanResult); + } + } + + /** + * Resolve the GitHub context needed for check run API calls. + */ + private async resolveCheckRunContext( + scanResult: GitHubScanResultRecord, + ): Promise<{ installationId: number; owner: string; repo: string }> { + const repoRecord = await this.repository.findRepositoryById(scanResult.repositoryId); + if (!repoRecord) throw new Error(`Repository ${scanResult.repositoryId} not found`); + + const installation = await this.repository.findInstallationById(repoRecord.installationId); + if (!installation) throw new Error(`Installation ${repoRecord.installationId} not found`); + + const [owner, repo] = repoRecord.fullName.split('/'); + return { installationId: installation.installationId, owner, repo }; + } + + /** + * Handle check run update errors: + * - 404/410: check run deleted, clear checkRunId and continue + * - transient/5xx: keep scan running, retry next poll + */ + private async handleCheckRunError( + err: unknown, + scanResult: GitHubScanResultRecord, + ): Promise { + const errorMessage = err instanceof Error ? err.message : String(err); + + // Check for 404/410 — check run was deleted + if (errorMessage.includes('404') || errorMessage.includes('410')) { + this.logger.warn( + `Check run ${scanResult.checkRunId} not found, clearing checkRunId for scan ${scanResult.id}`, + ); + await this.repository.updateScanResult(scanResult.id, { checkRunId: null }); + return true; // Proceed with terminal status + } + + // Transient error — keep running, retry next poll + this.logger.warn(`Transient check run update error for scan ${scanResult.id}: ${errorMessage}`); + return false; + } + + private async postPrReviewIfEnabled( + scanResult: GitHubScanResultRecord, + findings: Record[], + ): Promise { + if (scanResult.prReviewId) return; + if (scanResult.prNumber == null) return; + if (!scanResult.triggerRuleId) return; + + const rule = await this.repository.findTriggerRuleById(scanResult.triggerRuleId); + if (!rule?.postPrReview) return; + + const validFindings = this.toReviewFindingSummaries(findings); + if (validFindings.length === 0) return; + + const { installationId, owner, repo } = await this.resolveCheckRunContext(scanResult); + const prFiles = await this.githubAppService.getPullRequestFiles( + installationId, + owner, + repo, + scanResult.prNumber, + ); + const patchMap = buildPatchMap(prFiles); + + const reviewComments: { path: string; line: number; side: 'RIGHT'; body: string }[] = []; + const skippedFindings: ReviewFindingSummary[] = []; + const cappedFindings: ReviewFindingSummary[] = []; + + for (const finding of validFindings) { + const path = stripWorkspacePrefix(finding.file); + const patch = patchMap.get(path); + if (!patch || !isLineInDiffHunks(finding.line, patch)) { + skippedFindings.push({ ...finding, file: path }); + continue; + } + + if (reviewComments.length >= MAX_REVIEW_INLINE_COMMENTS) { + cappedFindings.push({ ...finding, file: path }); + continue; + } + + reviewComments.push({ + path, + line: finding.line, + side: 'RIGHT', + body: this.formatReviewCommentBody(finding), + }); + } + + if ( + reviewComments.length === 0 && + skippedFindings.length === 0 && + cappedFindings.length === 0 + ) { + return; + } + if (!scanResult.commitSha) { + this.logger.warn(`Skipping PR review for scan ${scanResult.id}: missing commit SHA`); + return; + } + + const summary = this.summarizeFindings(findings); + const reviewBody = this.buildReviewBody({ + summary, + inlineCount: reviewComments.length, + skippedFindings, + cappedFindingsCount: cappedFindings.length, + }); + + const firstAttempt = await this.githubAppService.createPullRequestReview( + installationId, + owner, + repo, + scanResult.prNumber, + { + commit_id: scanResult.commitSha, + body: reviewBody, + event: 'COMMENT', + comments: reviewComments, + }, + ); + + if ('status' in firstAttempt && firstAttempt.status === 422) { + const fallbackBody = this.buildReviewBody({ + summary, + inlineCount: 0, + skippedFindings: validFindings, + cappedFindingsCount: 0, + }); + + try { + const fallback = await this.githubAppService.createPullRequestReview( + installationId, + owner, + repo, + scanResult.prNumber, + { + commit_id: scanResult.commitSha, + body: fallbackBody, + event: 'COMMENT', + comments: [], + }, + ); + + if ('status' in fallback) { + this.logger.warn( + `Fallback PR review failed with 422 for scan ${scanResult.id}: ${fallback.error}`, + ); + return; + } + + await this.repository.updateScanResult(scanResult.id, { prReviewId: fallback.id }); + return; + } catch (err) { + this.logger.warn( + `Fallback PR review failed for scan ${scanResult.id}: ${err instanceof Error ? err.message : String(err)}`, + ); + return; + } + } + + if ('status' in firstAttempt) { + return; + } + + await this.repository.updateScanResult(scanResult.id, { prReviewId: firstAttempt.id }); + } + + private toReviewFindingSummaries(findings: Record[]): ReviewFindingSummary[] { + const summaries: ReviewFindingSummary[] = []; + + for (const finding of findings) { + const file = finding.file; + const line = finding.line; + if (typeof file !== 'string' || file.trim().length === 0) continue; + if (typeof line !== 'number' || !Number.isInteger(line) || line < 1) continue; + + summaries.push({ + file: file.trim(), + line, + severity: this.normalizeSeverity(finding.severity), + message: + typeof finding.message === 'string' && finding.message.trim().length > 0 + ? finding.message + : 'Finding detected', + ruleId: typeof finding.ruleId === 'string' ? finding.ruleId : undefined, + category: typeof finding.category === 'string' ? finding.category : undefined, + snippet: typeof finding.snippet === 'string' ? finding.snippet : undefined, + }); + } + + return summaries; + } + + private summarizeFindings(findings: Record[]): ScanSummary { + const summary = { critical: 0, high: 0, medium: 0, low: 0, info: 0 }; + + for (const finding of findings) { + const severity = this.normalizeSeverity(finding.severity); + summary[severity] += 1; + } + + return summary; + } + + private formatReviewCommentBody(finding: ReviewFindingSummary): string { + const ruleId = escapeBackticks(finding.ruleId ?? 'unknown-rule'); + const scanner = escapeMarkdown(finding.category ?? 'unknown-scanner'); + const severity = finding.severity.toUpperCase(); + const lines = [ + `**:warning: ${severity}** | \`${ruleId}\` | ${scanner}`, + '', + escapeMarkdown(finding.message), + ]; + + if (finding.snippet) { + const snippet = safeTruncate(finding.snippet, MAX_REVIEW_SNIPPET_LENGTH).replace( + /```/g, + '``\\`', + ); + lines.push( + '', + '
Code snippet', + '', + '```', + snippet, + '```', + '', + '
', + ); + } + + return safeTruncate(lines.join('\n'), MAX_REVIEW_COMMENT_LENGTH); + } + + private buildReviewBody(params: { + summary: ScanSummary; + inlineCount: number; + skippedFindings: ReviewFindingSummary[]; + cappedFindingsCount: number; + }): string { + const lines = [ + '## ShipSec Security Review', + '', + '| Severity | Count |', + '|----------|-------|', + `| Critical | ${params.summary.critical} |`, + `| High | ${params.summary.high} |`, + `| Medium | ${params.summary.medium} |`, + `| Low | ${params.summary.low} |`, + `| Info | ${params.summary.info} |`, + '', + `**${params.inlineCount}** inline comments posted.`, + ]; + + if (params.cappedFindingsCount > 0) { + lines.push( + `**${params.cappedFindingsCount}** additional inline-eligible findings not posted (50 comment limit).`, + ); + } + + if (params.skippedFindings.length > 0) { + lines.push('', `**${params.skippedFindings.length}** findings outside the diff:`, ''); + + const listed = params.skippedFindings.slice(0, MAX_REVIEW_SKIPPED_IN_BODY); + for (const finding of listed) { + const file = escapeBackticks(finding.file); + const message = escapeMarkdown(finding.message); + lines.push( + `- \`${file}:${finding.line}\` - **${finding.severity.toUpperCase()}** ${message}`, + ); + } + + const remaining = params.skippedFindings.length - listed.length; + if (remaining > 0) { + lines.push(`- ... and ${remaining} more`); + } + } + + return safeTruncate(lines.join('\n'), MAX_REVIEW_BODY_LENGTH); + } + + /** + * Extract findings data from workflow result outputs. + * Walks all node outputs looking for normalized findings data, + * check run IDs, and PR comment IDs. Merges findings from + * multiple normalize nodes (e.g. trufflehog + opengrep). + */ + private extractScanDataFromResult(result: unknown): Partial { + const data: Partial = {}; + + // The workflow result is { outputs: Record, success: boolean } + const resultObj = result as Record | null; + const outputs = (resultObj?.outputs ?? {}) as Record; + + const mergedSummary = { critical: 0, high: 0, medium: 0, low: 0, info: 0 }; + const allFindings: Record[] = []; + let totalFindingCount = 0; + + for (const nodeOutput of Object.values(outputs)) { + if (!nodeOutput || typeof nodeOutput !== 'object') continue; + const output = nodeOutput as Record; + + // Only collect findings from normalize nodes (identified by having a + // summary object with severity counts). Raw scanner nodes also expose + // findings/findingCount, but including them would double-count. + const isNormalizeNode = + output.summary && + typeof output.summary === 'object' && + 'critical' in (output.summary as object); + + if (isNormalizeNode) { + const summary = output.summary as Record; + mergedSummary.critical += summary.critical ?? 0; + mergedSummary.high += summary.high ?? 0; + mergedSummary.medium += summary.medium ?? 0; + mergedSummary.low += summary.low ?? 0; + mergedSummary.info += summary.info ?? 0; + + if (typeof output.findingCount === 'number') { + totalFindingCount += output.findingCount; + } + + if (Array.isArray(output.findings)) { + for (const finding of output.findings) { + if (finding && typeof finding === 'object') { + allFindings.push(finding as Record); + } + } + } + } + + // Look for check run ID (from github.check.create/update) + if (typeof output.checkRunId === 'number') { + data.checkRunId = output.checkRunId; + } + + // Look for PR comment ID (from github.pr.comment) — last valid wins + if (typeof output.commentId === 'number') { + data.prCommentId = output.commentId; + } + } + + if (totalFindingCount > 0 || allFindings.length > 0) { + data.summary = mergedSummary; + data.findingsCount = Math.max(totalFindingCount, allFindings.length); + + // Map normalized findings to the scan result schema + data.findings = allFindings.map((f) => ({ + id: String(f.finding_hash ?? f.id ?? crypto.randomUUID()), + type: String(f.scanner ?? f.type ?? 'unknown'), + severity: this.normalizeSeverity(f.severity), + file: String(f.file ?? ''), + line: typeof f.line === 'number' ? f.line : undefined, + endLine: typeof f.endLine === 'number' ? f.endLine : undefined, + message: this.firstNonEmptyString(f.description, f.title, f.message), + snippet: typeof f.snippet === 'string' ? f.snippet : undefined, + ruleId: typeof f.rule_id === 'string' ? f.rule_id : undefined, + category: typeof f.scanner === 'string' ? f.scanner : undefined, + })); + } + + return data; + } + + private normalizeSeverity(value: unknown): 'critical' | 'high' | 'medium' | 'low' | 'info' { + const valid = ['critical', 'high', 'medium', 'low', 'info']; + const s = String(value).toLowerCase(); + return valid.includes(s) ? (s as 'critical' | 'high' | 'medium' | 'low' | 'info') : 'info'; + } + + private firstNonEmptyString(...values: unknown[]): string { + for (const value of values) { + if (typeof value !== 'string') continue; + if (value.trim().length === 0) continue; + return value; + } + return ''; + } +} diff --git a/backend/src/main.ts b/backend/src/main.ts index 8c8c42364..d56b0054f 100644 --- a/backend/src/main.ts +++ b/backend/src/main.ts @@ -13,6 +13,7 @@ async function bootstrap() { await enforceVersionCheck(); const app = await NestFactory.create(AppModule, { logger: ['log', 'error', 'warn'], + rawBody: true, }); // Enable cookie parsing for session auth diff --git a/backend/src/workflows/__tests__/workflows.service.spec.ts b/backend/src/workflows/__tests__/workflows.service.spec.ts index 3ccda0bd2..f3188b78e 100644 --- a/backend/src/workflows/__tests__/workflows.service.spec.ts +++ b/backend/src/workflows/__tests__/workflows.service.spec.ts @@ -295,9 +295,6 @@ describe('WorkflowsService', () => { async hasPendingInputs() { return false; }, - async cacheTerminalStatus() { - // no-op in tests - }, }; const traceRepositoryMock = { @@ -407,7 +404,6 @@ describe('WorkflowsService', () => { traceRepositoryMock as any, temporalService, analyticsServiceMock as any, - { record: vi.fn() } as any, ); }); @@ -558,6 +554,44 @@ describe('WorkflowsService', () => { expect(second.runId).toBe(first.runId); }); + it('applies componentParams to override action params by componentId', async () => { + await service.create(sampleGraph, authContext); + const definition = compileWorkflowGraph(sampleGraph); + repositoryMock.findById = async () => ({ + id: 'workflow-id', + createdAt: new Date(now), + updatedAt: new Date(now), + name: sampleGraph.name, + description: sampleGraph.description ?? null, + graph: sampleGraph, + compiledDefinition: definition, + lastRun: null, + runCount: 0, + organizationId: TEST_ORG, + }); + + const prepared = await service.prepareRunPayload( + 'workflow-id', + { inputs: { fileId: 'test-file' } }, + authContext, + { + componentParams: { + 'core.file.loader': { path: '/override/path', mode: 'stream' }, + }, + }, + ); + + // Find the loader action and verify params were merged + const loaderAction = prepared.definition.actions.find( + (a) => a.componentId === 'core.file.loader', + ); + expect(loaderAction).toBeDefined(); + expect(loaderAction!.params).toMatchObject({ + path: '/override/path', + mode: 'stream', + }); + }); + it('delegates status, result, and cancel operations to the Temporal service', async () => { await service.create(sampleGraph, authContext); const run = await service.run('workflow-id', {}, authContext); @@ -619,7 +653,6 @@ describe('WorkflowsService', () => { traceRepositoryMock as any, failureTemporalService, analyticsServiceMock as any, - { record: vi.fn() } as any, ); const versionRecord = createWorkflowVersionRecord('workflow-id'); diff --git a/backend/src/workflows/workflows.module.ts b/backend/src/workflows/workflows.module.ts index 01332d0ed..86ca96b76 100644 --- a/backend/src/workflows/workflows.module.ts +++ b/backend/src/workflows/workflows.module.ts @@ -26,6 +26,8 @@ import { WorkflowRoleGuard } from './workflow-role.guard'; TerminalModule, AnalyticsModule, NodeIOModule, + // GitHubAppModule is @Global() — no import needed here. + // WorkflowsService accesses it via @Optional() @Inject('GitHubAppService'). ], controllers: [WorkflowsController, InternalRunsController], providers: [ diff --git a/backend/src/workflows/workflows.service.ts b/backend/src/workflows/workflows.service.ts index 402c349e1..1b508aaf3 100644 --- a/backend/src/workflows/workflows.service.ts +++ b/backend/src/workflows/workflows.service.ts @@ -3,6 +3,8 @@ import { randomUUID, createHash } from 'node:crypto'; import { Injectable, Logger, + Optional, + Inject, NotFoundException, ForbiddenException, BadRequestException, @@ -34,7 +36,6 @@ import { WorkflowRunRepository } from './repository/workflow-run.repository'; import { WorkflowVersionRepository } from './repository/workflow-version.repository'; import { TraceRepository } from '../trace/trace.repository'; import { AnalyticsService } from '../analytics/analytics.service'; -import { AuditLogService } from '../audit/audit-log.service'; import { ExecutionStatus, FailureSummary, @@ -44,24 +45,10 @@ import { ExecutionTriggerType, ExecutionInputPreview, ExecutionTriggerMetadata, - TERMINAL_STATUSES, } from '@shipsec/shared'; import type { WorkflowRunRecord, WorkflowVersionRecord, WorkflowGraph } from '../database/schema'; import type { AuthContext } from '../auth/types'; -export interface WorkflowSummaryResponse { - id: string; - name: string; - description: string | null; - organizationId: string | null; - lastRun: string | null; - latestRunStatus: string | null; - runCount: number; - nodeCount: number; - createdAt: string; - updatedAt: string; -} - export interface WorkflowRunRequest { inputs?: Record; versionId?: string; @@ -155,7 +142,12 @@ export class WorkflowsService { private readonly traceRepository: TraceRepository, private readonly temporalService: TemporalService, private readonly analyticsService: AnalyticsService, - private readonly auditLogService: AuditLogService, + @Optional() + @Inject('GitHubAppService') + private readonly githubAppService?: { + isConfigured(): boolean; + resolveInstallationForRepo(repoFullName: string): Promise<{ installationId: number } | null>; + }, ) {} private resolveOrganizationId(auth?: AuthContext | null): string | null { @@ -322,17 +314,6 @@ export class WorkflowsService { this.logger.log( `Created workflow ${response.id} version ${version.version} (nodes=${input.nodes.length}, edges=${input.edges.length})`, ); - this.auditLogService.record(auth ?? null, { - action: 'workflow.create', - resourceType: 'workflow', - resourceId: response.id, - resourceName: response.name, - metadata: { - nodeCount: input.nodes.length, - edgeCount: input.edges.length, - version: version.version, - }, - }); return response; } @@ -364,17 +345,6 @@ export class WorkflowsService { this.logger.log( `Updated workflow ${response.id} to version ${version.version} (nodes=${input.nodes.length}, edges=${input.edges.length})`, ); - this.auditLogService.record(auth ?? null, { - action: 'workflow.update', - resourceType: 'workflow', - resourceId: response.id, - resourceName: response.name, - metadata: { - nodeCount: input.nodes.length, - edgeCount: input.edges.length, - version: version.version, - }, - }); return response; } @@ -392,15 +362,6 @@ export class WorkflowsService { const version = await this.versionRepository.findLatestByWorkflowId(id, { organizationId }); const response = this.buildWorkflowResponse(record, version ?? null); this.logger.log(`Updated workflow ${response.id} metadata (name=${dto.name})`); - this.auditLogService.record(auth ?? null, { - action: 'workflow.update_metadata', - resourceType: 'workflow', - resourceId: response.id, - resourceName: response.name, - metadata: { - name: dto.name, - }, - }); return response; } @@ -569,15 +530,8 @@ export class WorkflowsService { async delete(id: string, auth?: AuthContext | null): Promise { const organizationId = await this.requireWorkflowAdmin(id, auth); - const existing = await this.repository.findById(id, { organizationId }).catch(() => null); await this.repository.delete(id, { organizationId }); this.logger.log(`Deleted workflow ${id}`); - this.auditLogService.record(auth ?? null, { - action: 'workflow.delete', - resourceType: 'workflow', - resourceId: id, - resourceName: (existing as any)?.name ?? null, - }); } async list(auth?: AuthContext | null): Promise { @@ -595,18 +549,6 @@ export class WorkflowsService { return responses; } - async listSummary(auth?: AuthContext | null): Promise { - const organizationId = this.requireOrganizationId(auth); - const records = await this.repository.listSummary({ organizationId }); - return records.map((record) => ({ - ...record, - lastRun: record.lastRun?.toISOString() ?? null, - latestRunStatus: record.latestRunStatus ?? null, - createdAt: record.createdAt.toISOString(), - updatedAt: record.updatedAt.toISOString(), - })); - } - private computeDuration(start: Date, end?: Date | null): number { const startTime = new Date(start).getTime(); const endTime = end ? new Date(end).getTime() : Date.now(); @@ -645,48 +587,28 @@ export class WorkflowsService { : this.computeDuration(run.createdAt, run.updatedAt); let currentStatus: ExecutionStatus = 'RUNNING'; - let resolvedCloseTime: string | null = null; - - // Cache-first: skip Temporal RPC for runs with a cached terminal status - if (run.status && (TERMINAL_STATUSES as readonly string[]).includes(run.status)) { - currentStatus = run.status as ExecutionStatus; - resolvedCloseTime = run.closeTime?.toISOString() ?? null; - } else { - try { - const desc = await this.temporalService.describeWorkflow({ - workflowId: run.runId, - runId: run.temporalRunId ?? undefined, + try { + const status = await this.temporalService.describeWorkflow({ + workflowId: run.runId, + runId: run.temporalRunId ?? undefined, + }); + currentStatus = this.normalizeStatus(status.status); + } catch (error) { + // If Temporal can't find the workflow, infer status from trace events + if (this.isNotFoundError(error)) { + currentStatus = this.inferStatusFromTraceEvents({ + runId: run.runId, + totalActions: run.totalActions ?? nodeCount, + completedActions, + failedActions, + startedActions, }); - currentStatus = this.normalizeStatus(desc.status); - resolvedCloseTime = desc.closeTime ?? null; - - // Cache terminal statuses (fire-and-forget) so future reads skip Temporal - if ((TERMINAL_STATUSES as readonly string[]).includes(currentStatus)) { - this.runRepository - .cacheTerminalStatus( - run.runId, - currentStatus, - desc.closeTime ? new Date(desc.closeTime) : undefined, - ) - .catch((err) => this.logger.warn(`Failed to cache status for ${run.runId}: ${err}`)); - } - } catch (error) { - // If Temporal can't find the workflow, infer status for display only — do NOT cache - if (this.isNotFoundError(error)) { - currentStatus = this.inferStatusFromTraceEvents({ - runId: run.runId, - totalActions: run.totalActions ?? nodeCount, - completedActions, - failedActions, - startedActions, - }); - this.logger.log( - `Run ${run.runId} not found in Temporal, inferred status: ${currentStatus} ` + - `(started=${startedActions}, completed=${completedActions}, failed=${failedActions})`, - ); - } else { - this.logger.warn(`Failed to get status for run ${run.runId}: ${error}`); - } + this.logger.log( + `Run ${run.runId} not found in Temporal, inferred status: ${currentStatus} ` + + `(started=${startedActions}, completed=${completedActions}, failed=${failedActions})`, + ); + } else { + this.logger.warn(`Failed to get status for run ${run.runId}: ${error}`); } } @@ -706,9 +628,7 @@ export class WorkflowsService { workflowVersion: run.workflowVersion ?? null, status: currentStatus, startTime: run.createdAt, - endTime: resolvedCloseTime - ? new Date(resolvedCloseTime) - : (run.closeTime ?? run.updatedAt ?? null), + endTime: run.updatedAt ?? null, temporalRunId: run.temporalRunId ?? undefined, workflowName, eventCount: startedActions, @@ -729,7 +649,6 @@ export class WorkflowsService { workflowId?: string; status?: ExecutionStatus; limit?: number; - offset?: number; } = {}, ) { const organizationId = this.requireOrganizationId(auth); @@ -817,17 +736,6 @@ export class WorkflowsService { this.logger.log( `Compiled workflow ${workflow.id} version ${version.version} with ${definition.actions.length} action(s); entrypoint=${definition.entrypoint.ref}`, ); - this.auditLogService.record(auth ?? null, { - action: 'workflow.commit', - resourceType: 'workflow', - resourceId: workflow.id, - resourceName: workflow.name, - metadata: { - version: version.version, - actionCount: definition.actions.length, - entrypoint: definition.entrypoint.ref, - }, - }); return definition; } @@ -841,6 +749,10 @@ export class WorkflowsService { string, { params?: Record; inputOverrides?: Record } >; + /** Inject inputOverrides into all actions matching a given componentId */ + componentInputs?: Record>; + /** Inject params into all actions matching a given componentId */ + componentParams?: Record>; runId?: string; idempotencyKey?: string; } = {}, @@ -848,24 +760,12 @@ export class WorkflowsService { const prepared = await this.prepareRunPayload(id, request, auth, { trigger: options.trigger, nodeOverrides: options.nodeOverrides, + componentInputs: options.componentInputs, + componentParams: options.componentParams, runId: options.runId, idempotencyKey: options.idempotencyKey, }); - this.auditLogService.record(auth ?? null, { - action: 'workflow.run', - resourceType: 'workflow', - resourceId: prepared.workflowId, - resourceName: null, - metadata: { - runId: prepared.runId, - workflowVersion: prepared.workflowVersion, - triggerType: options.trigger?.type ?? null, - triggerSourceId: options.trigger?.sourceId ?? null, - triggerLabel: options.trigger?.label ?? null, - }, - }); - return this.startPreparedRun(prepared); } @@ -1009,6 +909,10 @@ export class WorkflowsService { string, { params?: Record; inputOverrides?: Record } >; + /** Inject inputOverrides into all actions matching a given componentId */ + componentInputs?: Record>; + /** Inject params into all actions matching a given componentId */ + componentParams?: Record>; runId?: string; idempotencyKey?: string; parentRunId?: string; @@ -1031,6 +935,25 @@ export class WorkflowsService { const nodeOverrides = options.nodeOverrides ?? {}; let definitionWithOverrides = this.applyNodeOverrides(compiledDefinition, nodeOverrides); + // Auto-resolve GitHub installationId when not explicitly provided + const autoGitHubInputs = await this.resolveGitHubComponentInputs( + definitionWithOverrides, + options.componentInputs, + ); + const mergedComponentInputs = autoGitHubInputs + ? { ...autoGitHubInputs, ...options.componentInputs } + : options.componentInputs; + + definitionWithOverrides = this.applyComponentInputs( + definitionWithOverrides, + mergedComponentInputs, + ); + + definitionWithOverrides = this.applyComponentParams( + definitionWithOverrides, + options.componentParams, + ); + // Inject retry policies from component registry definitionWithOverrides = { ...definitionWithOverrides, @@ -1189,111 +1112,69 @@ export class WorkflowsService { let completedActions = 0; let failedActions = 0; let startedActions = 0; - let statusPayload: WorkflowRunStatusPayload; - - // Cache HIT — skip Temporal entirely for terminal runs - if (run.status && (TERMINAL_STATUSES as readonly string[]).includes(run.status)) { - // Still need completed actions for progress - if (run.totalActions && run.totalActions > 0) { - completedActions = await this.traceRepository.countByType( - runId, - 'NODE_COMPLETED', - organizationId, - ); - } - statusPayload = { - runId, - workflowId: run.workflowId, - status: run.status as ExecutionStatus, - startedAt: run.createdAt.toISOString(), - updatedAt: run.updatedAt ? new Date(run.updatedAt).toISOString() : new Date().toISOString(), - completedAt: run.closeTime?.toISOString() ?? undefined, - taskQueue: '', - historyLength: 0, - progress: - run.totalActions && run.totalActions > 0 - ? { - completedActions: Math.min(completedActions, run.totalActions), - totalActions: run.totalActions, - } - : undefined, - }; - } else { - // Cache MISS — query Temporal - // Pre-fetch trace event counts for status inference - if (run.totalActions && run.totalActions > 0) { - [completedActions, failedActions, startedActions] = await Promise.all([ - this.traceRepository.countByType(runId, 'NODE_COMPLETED', organizationId), - this.traceRepository.countByType(runId, 'NODE_FAILED', organizationId), - this.traceRepository.countByType(runId, 'NODE_STARTED', organizationId), - ]); - } + // Pre-fetch trace event counts for status inference + if (run.totalActions && run.totalActions > 0) { + [completedActions, failedActions, startedActions] = await Promise.all([ + this.traceRepository.countByType(runId, 'NODE_COMPLETED', organizationId), + this.traceRepository.countByType(runId, 'NODE_FAILED', organizationId), + this.traceRepository.countByType(runId, 'NODE_STARTED', organizationId), + ]); + } - try { - temporalStatus = await this.temporalService.describeWorkflow({ - workflowId: runId, - runId: temporalRunId, + try { + temporalStatus = await this.temporalService.describeWorkflow({ + workflowId: runId, + runId: temporalRunId, + }); + } catch (error) { + // If Temporal can't find the workflow, infer status from trace events + if (this.isNotFoundError(error)) { + const inferredStatus = this.inferStatusFromTraceEvents({ + runId, + totalActions: run.totalActions ?? 0, + completedActions, + failedActions, + startedActions, }); - // Cache terminal statuses (fire-and-forget) - const normalizedStatus = this.normalizeStatus(temporalStatus.status); - if ((TERMINAL_STATUSES as readonly string[]).includes(normalizedStatus)) { - this.runRepository - .cacheTerminalStatus( - run.runId, - normalizedStatus, - temporalStatus.closeTime ? new Date(temporalStatus.closeTime) : undefined, - ) - .catch((err) => this.logger.warn(`Failed to cache status for ${run.runId}: ${err}`)); - } - } catch (error) { - // If Temporal can't find the workflow, infer status from trace events - if (this.isNotFoundError(error)) { - const inferredStatus = this.inferStatusFromTraceEvents({ - runId, - totalActions: run.totalActions ?? 0, - completedActions, - failedActions, - startedActions, - }); - - this.logger.log( - `Workflow ${runId} not found in Temporal, inferred status: ${inferredStatus} ` + - `(started=${startedActions}, completed=${completedActions}, failed=${failedActions}, total=${run.totalActions})`, - ); + this.logger.log( + `Workflow ${runId} not found in Temporal, inferred status: ${inferredStatus} ` + + `(started=${startedActions}, completed=${completedActions}, failed=${failedActions}, total=${run.totalActions})`, + ); - temporalStatus = { - workflowId: runId, - runId: temporalRunId ?? runId, - // Cast to WorkflowExecutionStatusName - normalizeStatus handles mapping - status: inferredStatus as unknown as typeof temporalStatus.status, - startTime: run.createdAt.toISOString(), - // Only set closeTime for terminal states that actually ran - closeTime: ['COMPLETED', 'FAILED'].includes(inferredStatus) - ? new Date().toISOString() - : undefined, - historyLength: 0, - taskQueue: '', - }; - } else { - throw error; - } + temporalStatus = { + workflowId: runId, + runId: temporalRunId ?? runId, + // Cast to WorkflowExecutionStatusName - normalizeStatus handles mapping + status: inferredStatus as unknown as typeof temporalStatus.status, + startTime: run.createdAt.toISOString(), + // Only set closeTime for terminal states that actually ran + closeTime: ['COMPLETED', 'FAILED'].includes(inferredStatus) + ? new Date().toISOString() + : undefined, + historyLength: 0, + taskQueue: '', + }; + } else { + throw error; } + } - statusPayload = this.mapTemporalStatus(runId, temporalStatus, run, completedActions); + const statusPayload = this.mapTemporalStatus(runId, temporalStatus, run, completedActions); - // Override running status if waiting for human input - if (statusPayload.status === 'RUNNING') { - const hasPendingInput = await this.runRepository.hasPendingInputs(runId); - if (hasPendingInput) { - statusPayload.status = 'AWAITING_INPUT'; - } + // Override running status if waiting for human input + if (statusPayload.status === 'RUNNING') { + const hasPendingInput = await this.runRepository.hasPendingInputs(runId); + if (hasPendingInput) { + statusPayload.status = 'AWAITING_INPUT'; } } // Track workflow completion/failure when status changes to terminal state - if ((TERMINAL_STATUSES as readonly string[]).includes(statusPayload.status)) { + if ( + ['COMPLETED', 'FAILED', 'CANCELLED', 'TERMINATED', 'TIMED_OUT'].includes(statusPayload.status) + ) { const startTime = run.createdAt; const endTime = statusPayload.completedAt ? new Date(statusPayload.completedAt) : new Date(); const durationMs = endTime.getTime() - startTime.getTime(); @@ -1612,6 +1493,139 @@ export class WorkflowsService { }; } + /** + * Inject inputOverrides into all actions that match a given componentId. + * Used by server-side trigger methods to pass values (like installationId) + * to all instances of a component type without requiring manual wiring + * in the workflow graph. + */ + private applyComponentInputs( + definition: WorkflowDefinition, + componentInputs?: Record>, + ): WorkflowDefinition { + if (!componentInputs || Object.keys(componentInputs).length === 0) { + return definition; + } + + const updatedActions = definition.actions.map((action) => { + const overrides = componentInputs[action.componentId]; + if (!overrides || Object.keys(overrides).length === 0) { + return action; + } + + return { + ...action, + inputOverrides: { + ...(action.inputOverrides ?? {}), + ...overrides, + }, + }; + }); + + return { + ...definition, + actions: updatedActions, + }; + } + + /** + * Inject params into all actions that match a given componentId. + * Used by trigger handlers to override component parameters (like + * repository, branch, ref) without requiring manual wiring in the + * workflow graph. + */ + private applyComponentParams( + definition: WorkflowDefinition, + componentParams?: Record>, + ): WorkflowDefinition { + if (!componentParams || Object.keys(componentParams).length === 0) { + return definition; + } + + const updatedActions = definition.actions.map((action) => { + const overrides = componentParams[action.componentId]; + if (!overrides || Object.keys(overrides).length === 0) { + return action; + } + + return { + ...action, + params: { ...(action.params ?? {}), ...overrides }, + }; + }); + + return { ...definition, actions: updatedActions }; + } + + private static readonly GITHUB_COMPONENT_IDS = [ + 'github.repo.clone', + 'github.pr.comment', + 'github.check.create', + 'github.check.update', + ] as const; + + /** + * Auto-detect GitHub components in the workflow and resolve the + * installation token from the repository configured in the clone-repo action. + * This ensures manual runs from the workflow editor UI can clone private repos. + */ + private async resolveGitHubComponentInputs( + definition: WorkflowDefinition, + existingComponentInputs?: Record>, + ): Promise> | undefined> { + if (!this.githubAppService || !this.githubAppService.isConfigured()) { + return undefined; + } + + // Check if any action uses a GitHub component + const hasGitHubComponents = definition.actions.some((action) => + WorkflowsService.GITHUB_COMPONENT_IDS.includes(action.componentId as any), + ); + if (!hasGitHubComponents) { + return undefined; + } + + // Skip if GitHub componentInputs are already provided (e.g. from triggerScan) + if (existingComponentInputs?.['github.repo.clone']?.installationId) { + return undefined; + } + + // Find the clone-repo action and extract the repository name + const cloneAction = definition.actions.find( + (action) => action.componentId === 'github.repo.clone', + ); + const repoName = + (cloneAction?.inputOverrides?.repository as string) ?? + (cloneAction?.params?.repository as string); + + if (!repoName) { + return undefined; + } + + try { + const resolved = await this.githubAppService.resolveInstallationForRepo(repoName); + if (!resolved) { + this.logger.debug(`GitHub repo ${repoName} not found in database, skipping auto-inject`); + return undefined; + } + + this.logger.log( + `Auto-injecting installationId ${resolved.installationId} for GitHub components (repo: ${repoName})`, + ); + + const inputs = { installationId: resolved.installationId }; + return { + 'github.repo.clone': inputs, + 'github.pr.comment': inputs, + 'github.check.create': inputs, + 'github.check.update': inputs, + }; + } catch (error) { + this.logger.warn(`Failed to auto-resolve GitHub installationId for ${repoName}: ${error}`); + return undefined; + } + } + private buildEntryPointTriggerMetadata(auth?: AuthContext | null): { type: ExecutionTriggerType; sourceId: string | null; diff --git a/docs/GITHUB-SECURITY-WORKFLOWS.md b/docs/GITHUB-SECURITY-WORKFLOWS.md new file mode 100644 index 000000000..805913c60 --- /dev/null +++ b/docs/GITHUB-SECURITY-WORKFLOWS.md @@ -0,0 +1,137 @@ +# GitHub Security Workflows + +This guide explains how to build robust security scanning workflows using the ShipSecAI GitHub component library. + +## Components Overview + +The library provides a set of composable components to handle the entire lifecycle of a security scan: + +| Category | Component | Purpose | +| ----------- | ----------------------------- | -------------------------------------------------------------- | +| **Context** | `github.pr.context` | Fetches PR metadata and changed files for targeted scanning. | +| **Setup** | `github.repo.clone` | Clones a repository into an isolated Docker volume. | +| **Scan** | `scanner.trufflehog` | Scans for secrets. | +| **Scan** | `scanner.opengrep` | Performs SAST analysis. | +| **Process** | `security.findings.normalize` | Standardizes findings from any scanner. | +| **Report** | `security.findings.markdown` | Formats findings into a GitHub Markdown report. | +| **Notify** | `github.pr.comment` | Posts the report as a PR comment. | +| **Status** | `github.check.create/update` | Reports status via GitHub Check Runs. | +| **Cleanup** | `github.repo.volume.cleanup` | **CRITICAL:** Removes the Docker volume to prevent disk leaks. | + +--- + +## Pattern 1: PR Comment Workflow + +This is the most common pattern: identifying security issues in a Pull Request and commenting on them. + +### Workflow Steps + +1. **Get PR Context**: Fetch changed files to filter results. +2. **Create Check Run**: value="in_progress" to mark the PR as being scanned. +3. **Clone Repository**: Clone the code into a volume. +4. **Scan**: Run one or more scanners (TruffleHog, OpenGrep, etc.) on the volume. +5. **Normalize**: specific scanner output -> standard format. +6. **Format**: standard format -> Markdown table. +7. **Comment**: Post the Markdown to the PR. +8. **Update Check Run**: value="completed" (success/failure). +9. **Cleanup**: Remove the volume. + +### Example Configuration + +#### 1. Get Context + +- **Component**: `github.pr.context` +- **Inputs**: `installationId`, `owner`, `repo`, `pullNumber` +- **Output**: `changedFiles`, `headSha` + +#### 2. Clone Repository + +- **Component**: `github.repo.clone` +- **Inputs**: `owner`, `repo`, `ref` (use `headSha` from context) +- **Output**: `volumePath`, `volumeName` + +#### 3. Scan (e.g., TruffleHog) + +- **Component**: `scanner.trufflehog` +- **Inputs**: `volumePath` +- **Output**: `findings` + +#### 4. Normalize & Filter + +- **Component**: `security.findings.normalize` +- **Inputs**: + - `rawFindings`: output from scanner + - `changedFiles`: output from PR context +- **Params**: `filterToChangedFiles: true` +- **Output**: `findings` (normalized) + +#### 5. Format Report + +- **Component**: `security.findings.markdown` +- **Inputs**: + - `findings`: normalized findings + - `prContext`: output from PR context (for file links) +- **Params**: `grouping: severity` +- **Output**: `markdown`, `hasFindings` + +#### 6. Post Comment + +- **Component**: `github.pr.comment` +- **Inputs**: `owner`, `repo`, `prNumber`, `body` (markdown) +- **Condition**: Only if `hasFindings` is true + +#### 7. Cleanup (Always Run) + +- **Component**: `github.repo.volume.cleanup` +- **Inputs**: `volumeName` (from clone step) + +--- + +## Pattern 2: Check Run Lifecycle + +Use GitHub Check Runs to block merges on security failures. + +1. **Start**: Call `github.check.create`. + - `status`: "in_progress" + - `name`: "Security Scan" + - **Result**: Save `checkRunId`. + +2. **Execute**: Run scans. Catch errors. + +3. **Finish**: Call `github.check.update`. + - `checkRunId`: from step 1 + - `status`: "completed" + - `conclusion`: "success" (no findings) or "failure" (findings found) + - `output`: { title, summary, text } (can use formatted markdown) + +--- + +## Best Practices + +### 1. Always Clean Up Volumes + +The `github.repo.clone` component creates substantial Docker volumes (repository size). If you do not run `github.repo.volume.cleanup`, the worker's disk will fill up. + +- **Structure**: Place cleanup at the end of the workflow. +- **Reliability**: Ensure it runs even if previous steps fail (use "Always Run" or error handling paths). + +### 2. Filter to Changed Files + +Scanning the entire repository on every PR is slow and noisy. + +- Use `github.pr.context` to get `changedFiles`. +- Pass this to `security.findings.normalize` with `filterToChangedFiles: true`. +- This ensures developers are only alerted about issues _they_ introduced. + +### 3. Rate Limits + +Be mindful of GitHub API rate limits when posting comments. + +- The `findings-markdown` component truncates very long reports. +- Consider posting only a summary if findings > 50. + +### 4. Security + +- **Isolation**: Cloned repos live in isolated, read-only Docker volumes. +- **Secrets**: Tokens are handled securely via `github-auth` and not logged. +- **Validation**: All inputs are validated with Zod schemas. diff --git a/docs/samples/01-opengrep-repo-scan.json b/docs/samples/01-opengrep-repo-scan.json new file mode 100644 index 000000000..7e3793e31 --- /dev/null +++ b/docs/samples/01-opengrep-repo-scan.json @@ -0,0 +1,134 @@ +{ + "name": "OpenGrep Repository Scan", + "description": "Clone a GitHub repository, run OpenGrep SAST scanner, and output normalized findings to console log.", + "graph": { + "nodes": [ + { + "id": "entry-point", + "type": "core.workflow.entrypoint", + "position": { + "x": -147.6231177094379, + "y": 129.01060445387066 + }, + "data": { + "label": "Entry Point", + "config": { + "params": { + "runtimeInputs": [] + }, + "inputOverrides": {} + } + } + }, + { + "id": "clone-repo", + "type": "github.repo.clone", + "position": { + "x": 150.95809255896324, + "y": 133.86973476997326 + }, + "data": { + "label": "Clone Repository", + "config": { + "params": { + "depth": 1, + "repository": "LuD1161/dvwa" + }, + "inputOverrides": {} + } + } + }, + { + "id": "opengrep-scanner", + "type": "scanner.opengrep", + "position": { + "x": 460.81614598308056, + "y": 193.57533868392647 + }, + "data": { + "label": "OpenGrep SAST Scanner", + "config": { + "params": { + "jobs": 4, + "ruleset": "auto", + "timeout": 300 + }, + "inputOverrides": {} + } + } + }, + { + "id": "console-log", + "type": "core.console.log", + "position": { + "x": 1116.3751311389549, + "y": 127.274208167111 + }, + "data": { + "label": "Console Log - OpenGrep Findings", + "config": { + "params": {}, + "inputOverrides": { + "label": "OpenGrep Normalized Findings" + } + } + } + }, + { + "id": "security-findings-normalize-1770512156026", + "type": "security.findings.normalize", + "position": { + "x": 803.2792055896874, + "y": 146.3357324139255 + }, + "data": { + "label": "Normalize Security Findings", + "config": { + "params": { + "scannerType": "auto", + "filterToChangedFiles": true + }, + "inputOverrides": {} + } + } + } + ], + "edges": [ + { + "id": "edge-clone-to-opengrep", + "source": "clone-repo", + "target": "opengrep-scanner", + "sourceHandle": "volumeName", + "targetHandle": "volumeName", + "type": "default" + }, + { + "id": "reactflow__edge-security-findings-normalize-1770512156026findings-console-logdata", + "source": "security-findings-normalize-1770512156026", + "target": "console-log", + "sourceHandle": "findings", + "targetHandle": "data", + "type": "default" + }, + { + "id": "reactflow__edge-opengrep-scannerfindings-security-findings-normalize-1770512156026rawFindings", + "source": "opengrep-scanner", + "target": "security-findings-normalize-1770512156026", + "sourceHandle": "findings", + "targetHandle": "rawFindings", + "type": "default" + } + ], + "viewport": { + "x": 0, + "y": 0, + "zoom": 1 + } + }, + "metadata": { + "workflowId": "06dbd965-6e89-44ac-8498-f5a35c0dc611", + "currentVersionId": "b4a0271f-7687-48c4-b8ff-3f10a70c929b", + "currentVersion": 2, + "exportedAt": "2026-02-08T00:56:26.631Z" + } +} diff --git a/docs/samples/02-trufflehog-secret-scan.json b/docs/samples/02-trufflehog-secret-scan.json new file mode 100644 index 000000000..ca6084622 --- /dev/null +++ b/docs/samples/02-trufflehog-secret-scan.json @@ -0,0 +1,98 @@ +{ + "name": "TruffleHog Secret Scan", + "description": "Clone a GitHub repository, run TruffleHog secret scanner, and output normalized findings to console log.", + "nodes": [ + { + "id": "entry-point", + "type": "core.workflow.entrypoint", + "position": { + "x": 300, + "y": 0 + }, + "data": { + "label": "Entry Point", + "config": { + "params": { + "runtimeInputs": [] + }, + "inputOverrides": {} + } + } + }, + { + "id": "clone-repo", + "type": "github.repo.clone", + "position": { + "x": 300, + "y": 200 + }, + "data": { + "label": "Clone Repository", + "config": { + "params": { + "repository": "owner/repo", + "depth": 0 + }, + "inputOverrides": {} + } + } + }, + { + "id": "trufflehog-scanner", + "type": "scanner.trufflehog", + "position": { + "x": 300, + "y": 400 + }, + "data": { + "label": "TruffleHog Secret Scanner", + "config": { + "params": { + "onlyVerified": false, + "maxDepth": 50, + "concurrency": 8 + }, + "inputOverrides": {} + } + } + }, + { + "id": "console-log", + "type": "core.console.log", + "position": { + "x": 300, + "y": 600 + }, + "data": { + "label": "Console Log - TruffleHog Findings", + "config": { + "params": {}, + "inputOverrides": { + "label": "TruffleHog Normalized Findings" + } + } + } + } + ], + "edges": [ + { + "id": "edge-clone-to-trufflehog", + "source": "clone-repo", + "target": "trufflehog-scanner", + "sourceHandle": "volumePath", + "targetHandle": "volumePath" + }, + { + "id": "edge-trufflehog-to-log", + "source": "trufflehog-scanner", + "target": "console-log", + "sourceHandle": "findings", + "targetHandle": "data" + } + ], + "viewport": { + "x": 0, + "y": 0, + "zoom": 0.75 + } +} diff --git a/docs/samples/03-security-scan-with-pr-comments.json b/docs/samples/03-security-scan-with-pr-comments.json new file mode 100644 index 000000000..342e44757 --- /dev/null +++ b/docs/samples/03-security-scan-with-pr-comments.json @@ -0,0 +1,263 @@ +{ + "name": "Security Scan with PR Comments", + "description": "Clone a repository, run both OpenGrep and TruffleHog scanners in parallel, log normalized findings, and post results as PR comments.", + "nodes": [ + { + "id": "entry-point", + "type": "core.workflow.entrypoint", + "position": { + "x": 300, + "y": 0 + }, + "data": { + "label": "Entry Point", + "config": { + "params": { + "runtimeInputs": [ + { + "id": "repository", + "label": "Repository", + "type": "text", + "required": true, + "description": "GitHub repository in owner/repo format" + }, + { + "id": "owner", + "label": "Owner", + "type": "text", + "required": true, + "description": "Repository owner (org or user)" + }, + { + "id": "repo", + "label": "Repo", + "type": "text", + "required": true, + "description": "Repository name" + }, + { + "id": "prNumber", + "label": "PR Number", + "type": "number", + "required": true, + "description": "Pull request number to comment on" + } + ] + }, + "inputOverrides": {} + } + } + }, + { + "id": "clone-repo", + "type": "github.repo.clone", + "position": { + "x": 300, + "y": 200 + }, + "data": { + "label": "Clone Repository", + "config": { + "params": { + "repository": "owner/repo", + "depth": 1 + }, + "inputOverrides": {} + } + } + }, + { + "id": "opengrep-scanner", + "type": "scanner.opengrep", + "position": { + "x": 100, + "y": 420 + }, + "data": { + "label": "OpenGrep SAST Scanner", + "config": { + "params": { + "ruleset": "auto", + "timeout": 300, + "jobs": 4 + }, + "inputOverrides": {} + } + } + }, + { + "id": "trufflehog-scanner", + "type": "scanner.trufflehog", + "position": { + "x": 520, + "y": 420 + }, + "data": { + "label": "TruffleHog Secret Scanner", + "config": { + "params": { + "onlyVerified": false, + "maxDepth": 50, + "concurrency": 8 + }, + "inputOverrides": {} + } + } + }, + { + "id": "opengrep-log", + "type": "core.console.log", + "position": { + "x": 100, + "y": 640 + }, + "data": { + "label": "Console Log - OpenGrep", + "config": { + "params": {}, + "inputOverrides": { + "label": "OpenGrep Normalized Findings" + } + } + } + }, + { + "id": "trufflehog-log", + "type": "core.console.log", + "position": { + "x": 520, + "y": 640 + }, + "data": { + "label": "Console Log - TruffleHog", + "config": { + "params": {}, + "inputOverrides": { + "label": "TruffleHog Normalized Findings" + } + } + } + }, + { + "id": "pr-comment-opengrep", + "type": "github.pr.comment", + "position": { + "x": 100, + "y": 860 + }, + "data": { + "label": "PR Comment - OpenGrep Results", + "config": { + "params": {}, + "inputOverrides": {} + } + } + }, + { + "id": "pr-comment-trufflehog", + "type": "github.pr.comment", + "position": { + "x": 520, + "y": 860 + }, + "data": { + "label": "PR Comment - TruffleHog Results", + "config": { + "params": {}, + "inputOverrides": {} + } + } + } + ], + "edges": [ + { + "id": "edge-clone-to-opengrep", + "source": "clone-repo", + "target": "opengrep-scanner", + "sourceHandle": "volumePath", + "targetHandle": "volumePath" + }, + { + "id": "edge-clone-to-trufflehog", + "source": "clone-repo", + "target": "trufflehog-scanner", + "sourceHandle": "volumePath", + "targetHandle": "volumePath" + }, + { + "id": "edge-opengrep-to-log", + "source": "opengrep-scanner", + "target": "opengrep-log", + "sourceHandle": "findings", + "targetHandle": "data" + }, + { + "id": "edge-trufflehog-to-log", + "source": "trufflehog-scanner", + "target": "trufflehog-log", + "sourceHandle": "findings", + "targetHandle": "data" + }, + { + "id": "edge-opengrep-to-pr-comment", + "source": "opengrep-scanner", + "target": "pr-comment-opengrep", + "sourceHandle": "rawOutput", + "targetHandle": "body" + }, + { + "id": "edge-trufflehog-to-pr-comment", + "source": "trufflehog-scanner", + "target": "pr-comment-trufflehog", + "sourceHandle": "rawOutput", + "targetHandle": "body" + }, + { + "id": "edge-entry-owner-to-opengrep-comment", + "source": "entry-point", + "target": "pr-comment-opengrep", + "sourceHandle": "owner", + "targetHandle": "owner" + }, + { + "id": "edge-entry-repo-to-opengrep-comment", + "source": "entry-point", + "target": "pr-comment-opengrep", + "sourceHandle": "repo", + "targetHandle": "repo" + }, + { + "id": "edge-entry-pr-to-opengrep-comment", + "source": "entry-point", + "target": "pr-comment-opengrep", + "sourceHandle": "prNumber", + "targetHandle": "prNumber" + }, + { + "id": "edge-entry-owner-to-trufflehog-comment", + "source": "entry-point", + "target": "pr-comment-trufflehog", + "sourceHandle": "owner", + "targetHandle": "owner" + }, + { + "id": "edge-entry-repo-to-trufflehog-comment", + "source": "entry-point", + "target": "pr-comment-trufflehog", + "sourceHandle": "repo", + "targetHandle": "repo" + }, + { + "id": "edge-entry-pr-to-trufflehog-comment", + "source": "entry-point", + "target": "pr-comment-trufflehog", + "sourceHandle": "prNumber", + "targetHandle": "prNumber" + } + ], + "viewport": { + "x": 0, + "y": 0, + "zoom": 0.65 + } +} diff --git a/docs/samples/04-pr-security-scan.json b/docs/samples/04-pr-security-scan.json new file mode 100644 index 000000000..f60aab234 --- /dev/null +++ b/docs/samples/04-pr-security-scan.json @@ -0,0 +1,436 @@ +{ + "name": "PR Security Scan", + "description": "Triggered on PR events. Fetches PR context, clones the repository at the PR head, runs OpenGrep and TruffleHog in parallel, normalizes findings, formats them as Markdown, and posts results as PR comments.", + "nodes": [ + { + "id": "entry-point", + "type": "core.workflow.entrypoint", + "position": { + "x": 300, + "y": 0 + }, + "data": { + "label": "Entry Point", + "config": { + "params": { + "runtimeInputs": [ + { + "id": "owner", + "label": "Owner", + "type": "text", + "required": true, + "description": "Repository owner (org or user)" + }, + { + "id": "repo", + "label": "Repo", + "type": "text", + "required": true, + "description": "Repository name" + }, + { + "id": "prNumber", + "label": "PR Number", + "type": "number", + "required": true, + "description": "Pull request number" + }, + { + "id": "installationId", + "label": "Installation ID", + "type": "number", + "required": false, + "description": "GitHub App installation ID (auto-resolved if not provided)" + } + ] + }, + "inputOverrides": {} + } + } + }, + { + "id": "pr-context", + "type": "github.pr.context", + "position": { + "x": 300, + "y": 200 + }, + "data": { + "label": "Get PR Context", + "config": { + "params": {}, + "inputOverrides": {} + } + } + }, + { + "id": "clone-repo", + "type": "github.repo.clone", + "position": { + "x": 300, + "y": 420 + }, + "data": { + "label": "Clone PR Branch", + "config": { + "params": { + "repository": "owner/repo", + "depth": 1 + }, + "inputOverrides": {} + } + } + }, + { + "id": "opengrep-scanner", + "type": "scanner.opengrep", + "position": { + "x": 100, + "y": 640 + }, + "data": { + "label": "OpenGrep SAST Scanner", + "config": { + "params": { + "ruleset": "auto", + "timeout": 300, + "jobs": 4 + }, + "inputOverrides": {} + } + } + }, + { + "id": "trufflehog-scanner", + "type": "scanner.trufflehog", + "position": { + "x": 520, + "y": 640 + }, + "data": { + "label": "TruffleHog Secret Scanner", + "config": { + "params": { + "onlyVerified": false, + "maxDepth": 50, + "concurrency": 8 + }, + "inputOverrides": {} + } + } + }, + { + "id": "opengrep-log", + "type": "core.console.log", + "position": { + "x": 0, + "y": 860 + }, + "data": { + "label": "Console Log - OpenGrep", + "config": { + "params": {}, + "inputOverrides": { + "label": "OpenGrep Findings" + } + } + } + }, + { + "id": "trufflehog-log", + "type": "core.console.log", + "position": { + "x": 420, + "y": 860 + }, + "data": { + "label": "Console Log - TruffleHog", + "config": { + "params": {}, + "inputOverrides": { + "label": "TruffleHog Findings" + } + } + } + }, + { + "id": "normalize-opengrep", + "type": "security.findings.normalize", + "position": { + "x": 100, + "y": 860 + }, + "data": { + "label": "Normalize OpenGrep Findings", + "config": { + "params": { + "scannerType": "opengrep", + "filterToChangedFiles": true + }, + "inputOverrides": {} + } + } + }, + { + "id": "normalize-trufflehog", + "type": "security.findings.normalize", + "position": { + "x": 520, + "y": 860 + }, + "data": { + "label": "Normalize TruffleHog Findings", + "config": { + "params": { + "scannerType": "trufflehog", + "filterToChangedFiles": true + }, + "inputOverrides": {} + } + } + }, + { + "id": "markdown-opengrep", + "type": "security.findings.markdown", + "position": { + "x": 100, + "y": 1060 + }, + "data": { + "label": "Format OpenGrep Report", + "config": { + "params": { + "title": "OpenGrep SAST Results", + "grouping": "severity", + "maxFindings": 25 + }, + "inputOverrides": {} + } + } + }, + { + "id": "markdown-trufflehog", + "type": "security.findings.markdown", + "position": { + "x": 520, + "y": 1060 + }, + "data": { + "label": "Format TruffleHog Report", + "config": { + "params": { + "title": "TruffleHog Secret Scan Results", + "grouping": "severity", + "maxFindings": 25 + }, + "inputOverrides": {} + } + } + }, + { + "id": "pr-comment-opengrep", + "type": "github.pr.comment", + "position": { + "x": 100, + "y": 1260 + }, + "data": { + "label": "PR Comment - OpenGrep Results", + "config": { + "params": {}, + "inputOverrides": {} + } + } + }, + { + "id": "pr-comment-trufflehog", + "type": "github.pr.comment", + "position": { + "x": 520, + "y": 1260 + }, + "data": { + "label": "PR Comment - TruffleHog Results", + "config": { + "params": {}, + "inputOverrides": {} + } + } + } + ], + "edges": [ + { + "id": "edge-entry-installationId-to-pr-context", + "source": "entry-point", + "target": "pr-context", + "sourceHandle": "installationId", + "targetHandle": "installationId" + }, + { + "id": "edge-entry-installationId-to-pr-comment-opengrep", + "source": "entry-point", + "target": "pr-comment-opengrep", + "sourceHandle": "installationId", + "targetHandle": "installationId" + }, + { + "id": "edge-entry-installationId-to-pr-comment-trufflehog", + "source": "entry-point", + "target": "pr-comment-trufflehog", + "sourceHandle": "installationId", + "targetHandle": "installationId" + }, + { + "id": "edge-entry-owner-to-pr-context", + "source": "entry-point", + "target": "pr-context", + "sourceHandle": "owner", + "targetHandle": "owner" + }, + { + "id": "edge-entry-repo-to-pr-context", + "source": "entry-point", + "target": "pr-context", + "sourceHandle": "repo", + "targetHandle": "repo" + }, + { + "id": "edge-entry-pr-to-pr-context", + "source": "entry-point", + "target": "pr-context", + "sourceHandle": "prNumber", + "targetHandle": "pullNumber" + }, + { + "id": "edge-clone-to-opengrep", + "source": "clone-repo", + "target": "opengrep-scanner", + "sourceHandle": "volumeName", + "targetHandle": "volumeName" + }, + { + "id": "edge-clone-to-trufflehog", + "source": "clone-repo", + "target": "trufflehog-scanner", + "sourceHandle": "volumeName", + "targetHandle": "volumeName" + }, + { + "id": "edge-opengrep-to-log", + "source": "opengrep-scanner", + "target": "opengrep-log", + "sourceHandle": "findings", + "targetHandle": "data" + }, + { + "id": "edge-trufflehog-to-log", + "source": "trufflehog-scanner", + "target": "trufflehog-log", + "sourceHandle": "findings", + "targetHandle": "data" + }, + { + "id": "edge-opengrep-findings-to-normalize", + "source": "opengrep-scanner", + "target": "normalize-opengrep", + "sourceHandle": "findings", + "targetHandle": "rawFindings" + }, + { + "id": "edge-pr-context-changed-files-to-normalize-opengrep", + "source": "pr-context", + "target": "normalize-opengrep", + "sourceHandle": "changedFiles", + "targetHandle": "changedFiles" + }, + { + "id": "edge-normalize-opengrep-to-markdown", + "source": "normalize-opengrep", + "target": "markdown-opengrep", + "sourceHandle": "findings", + "targetHandle": "findings" + }, + { + "id": "edge-markdown-opengrep-to-pr-comment", + "source": "markdown-opengrep", + "target": "pr-comment-opengrep", + "sourceHandle": "markdown", + "targetHandle": "body" + }, + { + "id": "edge-trufflehog-findings-to-normalize", + "source": "trufflehog-scanner", + "target": "normalize-trufflehog", + "sourceHandle": "findings", + "targetHandle": "rawFindings" + }, + { + "id": "edge-pr-context-changed-files-to-normalize-trufflehog", + "source": "pr-context", + "target": "normalize-trufflehog", + "sourceHandle": "changedFiles", + "targetHandle": "changedFiles" + }, + { + "id": "edge-normalize-trufflehog-to-markdown", + "source": "normalize-trufflehog", + "target": "markdown-trufflehog", + "sourceHandle": "findings", + "targetHandle": "findings" + }, + { + "id": "edge-markdown-trufflehog-to-pr-comment", + "source": "markdown-trufflehog", + "target": "pr-comment-trufflehog", + "sourceHandle": "markdown", + "targetHandle": "body" + }, + { + "id": "edge-entry-owner-to-opengrep-comment", + "source": "entry-point", + "target": "pr-comment-opengrep", + "sourceHandle": "owner", + "targetHandle": "owner" + }, + { + "id": "edge-entry-repo-to-opengrep-comment", + "source": "entry-point", + "target": "pr-comment-opengrep", + "sourceHandle": "repo", + "targetHandle": "repo" + }, + { + "id": "edge-entry-pr-to-opengrep-comment", + "source": "entry-point", + "target": "pr-comment-opengrep", + "sourceHandle": "prNumber", + "targetHandle": "prNumber" + }, + { + "id": "edge-entry-owner-to-trufflehog-comment", + "source": "entry-point", + "target": "pr-comment-trufflehog", + "sourceHandle": "owner", + "targetHandle": "owner" + }, + { + "id": "edge-entry-repo-to-trufflehog-comment", + "source": "entry-point", + "target": "pr-comment-trufflehog", + "sourceHandle": "repo", + "targetHandle": "repo" + }, + { + "id": "edge-entry-pr-to-trufflehog-comment", + "source": "entry-point", + "target": "pr-comment-trufflehog", + "sourceHandle": "prNumber", + "targetHandle": "prNumber" + } + ], + "viewport": { + "x": 0, + "y": 0, + "zoom": 0.55 + } +} diff --git a/docs/samples/05-pr-security-check-run.json b/docs/samples/05-pr-security-check-run.json new file mode 100644 index 000000000..3a8a7270a --- /dev/null +++ b/docs/samples/05-pr-security-check-run.json @@ -0,0 +1,418 @@ +{ + "name": "PR Security Check Run", + "description": "Triggered on PR events. Creates GitHub Check Runs, fetches PR context, clones the repo, runs OpenGrep and TruffleHog in parallel, then updates the check runs with results.", + "nodes": [ + { + "id": "entry-point", + "type": "core.workflow.entrypoint", + "position": { + "x": 300, + "y": 0 + }, + "data": { + "label": "Entry Point", + "config": { + "params": { + "runtimeInputs": [ + { + "id": "owner", + "label": "Owner", + "type": "text", + "required": true, + "description": "Repository owner (org or user)" + }, + { + "id": "repo", + "label": "Repo", + "type": "text", + "required": true, + "description": "Repository name" + }, + { + "id": "prNumber", + "label": "PR Number", + "type": "number", + "required": true, + "description": "Pull request number" + }, + { + "id": "installationId", + "label": "Installation ID", + "type": "number", + "required": false, + "description": "GitHub App installation ID (auto-resolved if not provided)" + } + ] + }, + "inputOverrides": {} + } + } + }, + { + "id": "pr-context", + "type": "github.pr.context", + "position": { + "x": 300, + "y": 200 + }, + "data": { + "label": "Get PR Context", + "config": { + "params": {}, + "inputOverrides": {} + } + } + }, + { + "id": "create-check-opengrep", + "type": "github.check.create", + "position": { + "x": 100, + "y": 420 + }, + "data": { + "label": "Create Check - OpenGrep", + "config": { + "params": {}, + "inputOverrides": { + "name": "OpenGrep SAST Scan", + "status": "in_progress" + } + } + } + }, + { + "id": "create-check-trufflehog", + "type": "github.check.create", + "position": { + "x": 520, + "y": 420 + }, + "data": { + "label": "Create Check - TruffleHog", + "config": { + "params": {}, + "inputOverrides": { + "name": "TruffleHog Secret Scan", + "status": "in_progress" + } + } + } + }, + { + "id": "clone-repo", + "type": "github.repo.clone", + "position": { + "x": 300, + "y": 640 + }, + "data": { + "label": "Clone PR Branch", + "config": { + "params": { + "repository": "owner/repo", + "depth": 1 + }, + "inputOverrides": {} + } + } + }, + { + "id": "opengrep-scanner", + "type": "scanner.opengrep", + "position": { + "x": 100, + "y": 860 + }, + "data": { + "label": "OpenGrep SAST Scanner", + "config": { + "params": { + "ruleset": "auto", + "timeout": 300, + "jobs": 4 + }, + "inputOverrides": {} + } + } + }, + { + "id": "trufflehog-scanner", + "type": "scanner.trufflehog", + "position": { + "x": 520, + "y": 860 + }, + "data": { + "label": "TruffleHog Secret Scanner", + "config": { + "params": { + "onlyVerified": false, + "maxDepth": 50, + "concurrency": 8 + }, + "inputOverrides": {} + } + } + }, + { + "id": "opengrep-log", + "type": "core.console.log", + "position": { + "x": 100, + "y": 1080 + }, + "data": { + "label": "Console Log - OpenGrep", + "config": { + "params": {}, + "inputOverrides": { + "label": "OpenGrep Findings" + } + } + } + }, + { + "id": "trufflehog-log", + "type": "core.console.log", + "position": { + "x": 520, + "y": 1080 + }, + "data": { + "label": "Console Log - TruffleHog", + "config": { + "params": {}, + "inputOverrides": { + "label": "TruffleHog Findings" + } + } + } + }, + { + "id": "update-check-opengrep", + "type": "github.check.update", + "position": { + "x": 100, + "y": 1300 + }, + "data": { + "label": "Update Check - OpenGrep", + "config": { + "params": {}, + "inputOverrides": { + "status": "completed", + "conclusion": "neutral" + } + } + } + }, + { + "id": "update-check-trufflehog", + "type": "github.check.update", + "position": { + "x": 520, + "y": 1300 + }, + "data": { + "label": "Update Check - TruffleHog", + "config": { + "params": {}, + "inputOverrides": { + "status": "completed", + "conclusion": "neutral" + } + } + } + } + ], + "edges": [ + { + "id": "edge-entry-installationId-to-pr-context", + "source": "entry-point", + "target": "pr-context", + "sourceHandle": "installationId", + "targetHandle": "installationId" + }, + { + "id": "edge-entry-installationId-to-check-opengrep", + "source": "entry-point", + "target": "create-check-opengrep", + "sourceHandle": "installationId", + "targetHandle": "installationId" + }, + { + "id": "edge-entry-installationId-to-check-trufflehog", + "source": "entry-point", + "target": "create-check-trufflehog", + "sourceHandle": "installationId", + "targetHandle": "installationId" + }, + { + "id": "edge-entry-installationId-to-update-opengrep", + "source": "entry-point", + "target": "update-check-opengrep", + "sourceHandle": "installationId", + "targetHandle": "installationId" + }, + { + "id": "edge-entry-installationId-to-update-trufflehog", + "source": "entry-point", + "target": "update-check-trufflehog", + "sourceHandle": "installationId", + "targetHandle": "installationId" + }, + { + "id": "edge-entry-owner-to-pr-context", + "source": "entry-point", + "target": "pr-context", + "sourceHandle": "owner", + "targetHandle": "owner" + }, + { + "id": "edge-entry-repo-to-pr-context", + "source": "entry-point", + "target": "pr-context", + "sourceHandle": "repo", + "targetHandle": "repo" + }, + { + "id": "edge-entry-pr-to-pr-context", + "source": "entry-point", + "target": "pr-context", + "sourceHandle": "prNumber", + "targetHandle": "pullNumber" + }, + { + "id": "edge-pr-context-sha-to-check-opengrep", + "source": "pr-context", + "target": "create-check-opengrep", + "sourceHandle": "headSha", + "targetHandle": "headSha" + }, + { + "id": "edge-entry-owner-to-check-opengrep", + "source": "entry-point", + "target": "create-check-opengrep", + "sourceHandle": "owner", + "targetHandle": "owner" + }, + { + "id": "edge-entry-repo-to-check-opengrep", + "source": "entry-point", + "target": "create-check-opengrep", + "sourceHandle": "repo", + "targetHandle": "repo" + }, + { + "id": "edge-pr-context-sha-to-check-trufflehog", + "source": "pr-context", + "target": "create-check-trufflehog", + "sourceHandle": "headSha", + "targetHandle": "headSha" + }, + { + "id": "edge-entry-owner-to-check-trufflehog", + "source": "entry-point", + "target": "create-check-trufflehog", + "sourceHandle": "owner", + "targetHandle": "owner" + }, + { + "id": "edge-entry-repo-to-check-trufflehog", + "source": "entry-point", + "target": "create-check-trufflehog", + "sourceHandle": "repo", + "targetHandle": "repo" + }, + { + "id": "edge-clone-to-opengrep", + "source": "clone-repo", + "target": "opengrep-scanner", + "sourceHandle": "volumePath", + "targetHandle": "volumePath" + }, + { + "id": "edge-clone-to-trufflehog", + "source": "clone-repo", + "target": "trufflehog-scanner", + "sourceHandle": "volumePath", + "targetHandle": "volumePath" + }, + { + "id": "edge-opengrep-to-log", + "source": "opengrep-scanner", + "target": "opengrep-log", + "sourceHandle": "findings", + "targetHandle": "data" + }, + { + "id": "edge-trufflehog-to-log", + "source": "trufflehog-scanner", + "target": "trufflehog-log", + "sourceHandle": "findings", + "targetHandle": "data" + }, + { + "id": "edge-create-check-opengrep-to-update", + "source": "create-check-opengrep", + "target": "update-check-opengrep", + "sourceHandle": "checkRunId", + "targetHandle": "checkRunId" + }, + { + "id": "edge-entry-owner-to-update-opengrep", + "source": "entry-point", + "target": "update-check-opengrep", + "sourceHandle": "owner", + "targetHandle": "owner" + }, + { + "id": "edge-entry-repo-to-update-opengrep", + "source": "entry-point", + "target": "update-check-opengrep", + "sourceHandle": "repo", + "targetHandle": "repo" + }, + { + "id": "edge-opengrep-findings-to-update-check", + "source": "opengrep-scanner", + "target": "update-check-opengrep", + "sourceHandle": "rawOutput", + "targetHandle": "output" + }, + { + "id": "edge-create-check-trufflehog-to-update", + "source": "create-check-trufflehog", + "target": "update-check-trufflehog", + "sourceHandle": "checkRunId", + "targetHandle": "checkRunId" + }, + { + "id": "edge-entry-owner-to-update-trufflehog", + "source": "entry-point", + "target": "update-check-trufflehog", + "sourceHandle": "owner", + "targetHandle": "owner" + }, + { + "id": "edge-entry-repo-to-update-trufflehog", + "source": "entry-point", + "target": "update-check-trufflehog", + "sourceHandle": "repo", + "targetHandle": "repo" + }, + { + "id": "edge-trufflehog-findings-to-update-check", + "source": "trufflehog-scanner", + "target": "update-check-trufflehog", + "sourceHandle": "rawOutput", + "targetHandle": "output" + } + ], + "viewport": { + "x": 0, + "y": 0, + "zoom": 0.45 + } +} diff --git a/docs/samples/06-opengrep-pr-repo-scan.json b/docs/samples/06-opengrep-pr-repo-scan.json new file mode 100644 index 000000000..a06cbf1e6 --- /dev/null +++ b/docs/samples/06-opengrep-pr-repo-scan.json @@ -0,0 +1,229 @@ +{ + "name": "OpenGrep PR & Repo Scanner", + "description": "Run OpenGrep SAST scanner on a GitHub repository or PR. Fetches PR context, clones the repo at the PR head, runs OpenGrep with security-audit rulesets, logs findings, and posts results as a PR comment.", + "nodes": [ + { + "id": "entry-point", + "type": "core.workflow.entrypoint", + "position": { + "x": 300, + "y": 0 + }, + "data": { + "label": "Entry Point", + "config": { + "params": { + "runtimeInputs": [ + { + "id": "owner", + "label": "Owner", + "type": "text", + "required": true, + "description": "Repository owner (org or user)" + }, + { + "id": "repo", + "label": "Repo", + "type": "text", + "required": true, + "description": "Repository name" + }, + { + "id": "prNumber", + "label": "PR Number", + "type": "number", + "required": true, + "description": "Pull request number to scan" + }, + { + "id": "installationId", + "label": "Installation ID", + "type": "number", + "required": false, + "description": "GitHub App installation ID (auto-resolved if not provided)" + } + ] + }, + "inputOverrides": {} + } + } + }, + { + "id": "pr-context", + "type": "github.pr.context", + "position": { + "x": 300, + "y": 200 + }, + "data": { + "label": "Get PR Context", + "config": { + "params": {}, + "inputOverrides": {} + } + } + }, + { + "id": "clone-repo", + "type": "github.repo.clone", + "position": { + "x": 300, + "y": 420 + }, + "data": { + "label": "Clone PR Branch", + "config": { + "params": { + "repository": "owner/repo", + "depth": 1 + }, + "inputOverrides": {} + } + } + }, + { + "id": "opengrep-scanner", + "type": "scanner.opengrep", + "position": { + "x": 300, + "y": 640 + }, + "data": { + "label": "OpenGrep SAST Scanner", + "config": { + "params": { + "ruleset": "auto", + "severityFilter": ["ERROR", "WARNING"], + "excludePaths": ["node_modules", "vendor", ".git", "dist", "build"], + "timeout": 300, + "jobs": 4 + }, + "inputOverrides": {} + } + } + }, + { + "id": "console-log", + "type": "core.console.log", + "position": { + "x": 520, + "y": 860 + }, + "data": { + "label": "Console Log - OpenGrep Findings", + "config": { + "params": {}, + "inputOverrides": { + "label": "OpenGrep SAST Findings" + } + } + } + }, + { + "id": "pr-comment", + "type": "github.pr.comment", + "position": { + "x": 100, + "y": 860 + }, + "data": { + "label": "PR Comment - OpenGrep Results", + "config": { + "params": {}, + "inputOverrides": {} + } + } + } + ], + "edges": [ + { + "id": "edge-entry-installationId-to-pr-context", + "source": "entry-point", + "target": "pr-context", + "sourceHandle": "installationId", + "targetHandle": "installationId" + }, + { + "id": "edge-entry-owner-to-pr-context", + "source": "entry-point", + "target": "pr-context", + "sourceHandle": "owner", + "targetHandle": "owner" + }, + { + "id": "edge-entry-repo-to-pr-context", + "source": "entry-point", + "target": "pr-context", + "sourceHandle": "repo", + "targetHandle": "repo" + }, + { + "id": "edge-entry-pr-to-pr-context", + "source": "entry-point", + "target": "pr-context", + "sourceHandle": "prNumber", + "targetHandle": "pullNumber" + }, + { + "id": "edge-entry-installationId-to-clone", + "source": "entry-point", + "target": "clone-repo", + "sourceHandle": "installationId", + "targetHandle": "installationId" + }, + { + "id": "edge-clone-to-opengrep", + "source": "clone-repo", + "target": "opengrep-scanner", + "sourceHandle": "volumeName", + "targetHandle": "volumeName" + }, + { + "id": "edge-opengrep-to-log", + "source": "opengrep-scanner", + "target": "console-log", + "sourceHandle": "findings", + "targetHandle": "data" + }, + { + "id": "edge-opengrep-to-pr-comment", + "source": "opengrep-scanner", + "target": "pr-comment", + "sourceHandle": "rawOutput", + "targetHandle": "body" + }, + { + "id": "edge-entry-owner-to-pr-comment", + "source": "entry-point", + "target": "pr-comment", + "sourceHandle": "owner", + "targetHandle": "owner" + }, + { + "id": "edge-entry-repo-to-pr-comment", + "source": "entry-point", + "target": "pr-comment", + "sourceHandle": "repo", + "targetHandle": "repo" + }, + { + "id": "edge-entry-pr-to-pr-comment", + "source": "entry-point", + "target": "pr-comment", + "sourceHandle": "prNumber", + "targetHandle": "prNumber" + }, + { + "id": "edge-entry-installationId-to-pr-comment", + "source": "entry-point", + "target": "pr-comment", + "sourceHandle": "installationId", + "targetHandle": "installationId" + } + ], + "viewport": { + "x": 0, + "y": 0, + "zoom": 0.55 + } +} diff --git a/docs/samples/07-ai-code-review.json b/docs/samples/07-ai-code-review.json new file mode 100644 index 000000000..61dac0817 --- /dev/null +++ b/docs/samples/07-ai-code-review.json @@ -0,0 +1,200 @@ +{ + "name": "AI Code Review", + "description": "Review pull request code changes using OpenCode agent via Z.AI GLM Coding Plan and post results as a PR comment.", + "nodes": [ + { + "id": "entry-point", + "type": "core.workflow.entrypoint", + "position": { "x": 400, "y": 0 }, + "data": { + "label": "Entry Point", + "config": { + "params": { + "runtimeInputs": [ + { + "id": "owner", + "label": "Owner", + "type": "text", + "required": true, + "description": "Repository owner (org or user)" + }, + { + "id": "repo", + "label": "Repo", + "type": "text", + "required": true, + "description": "Repository name" + }, + { + "id": "prNumber", + "label": "PR Number", + "type": "number", + "required": true, + "description": "Pull request number to review" + }, + { + "id": "repository", + "label": "Repository", + "type": "text", + "required": true, + "description": "Full repository in owner/repo format" + }, + { + "id": "installationId", + "label": "Installation ID", + "type": "number", + "required": false, + "description": "GitHub App installation ID (auto-injected by trigger)" + } + ] + }, + "inputOverrides": {} + } + } + }, + { + "id": "clone-repo", + "type": "github.repo.clone", + "position": { "x": 400, "y": 200 }, + "data": { + "label": "Clone Repository", + "config": { + "params": { + "depth": 0 + }, + "inputOverrides": {} + } + } + }, + { + "id": "pr-context", + "type": "github.pr.context", + "position": { "x": 700, "y": 200 }, + "data": { + "label": "Get PR Context", + "config": { + "params": {}, + "inputOverrides": {} + } + } + }, + { + "id": "opencode-agent", + "type": "core.ai.opencode", + "position": { "x": 400, "y": 500 }, + "data": { + "label": "OpenCode Code Review Agent", + "config": { + "params": { + "systemPrompt": "You are an expert code reviewer. Your job is to review pull request changes thoroughly and provide constructive, actionable feedback.\n\nReview Guidelines:\n1. Focus on code quality, correctness, security, and best practices\n2. Look for bugs, logic errors, and edge cases\n3. Check for security vulnerabilities (injection, XSS, auth issues, etc.)\n4. Evaluate error handling and input validation\n5. Assess code style and readability\n6. Identify performance concerns\n7. Suggest improvements with specific code examples when helpful\n\nOutput Format:\nGenerate a well-structured markdown report. Start directly with the report — do NOT include any preamble, greetings, or debug output.\n\n## Code Review Report\n\n### Summary\nBrief overview of what the PR does (1-2 sentences).\n\nThen immediately include a **Findings Summary** table:\n\n| | Severity | File | Line | Finding |\n|---|----------|------|------|---------|\n| \ud83d\udd34 | Critical | `filename.py` | 42 | One-line description |\n| \ud83d\udfe0 | Warning | `filename.py` | 15 | One-line description |\n| \ud83d\udfe1 | Suggestion | `filename.py` | 8 | One-line description |\n| \ud83d\udd35 | Info | `filename.py` | — | One-line description |\n\nSeverity icons: \ud83d\udd34 Critical, \ud83d\udfe0 Warning, \ud83d\udfe1 Suggestion, \ud83d\udd35 Info\n\n### Detailed Findings\nFor each finding, provide:\n- Severity and title\n- Affected file and line number(s)\n- What the issue is and why it matters\n- Recommended fix with a short code snippet if helpful\n\n### Overall Assessment\nOne of: \u2705 Approve / \u274c Request Changes / \ud83d\udcac Comment\n\nBe concise but thorough. Focus on the changed files only. Output ONLY the markdown report — no debug info, no preamble.", + "autoApprove": true + }, + "inputOverrides": { + "task": "Review the pull request code changes provided in the context. Analyze each changed file's diff/patch and provide a thorough code review. Focus on bugs, security issues, code quality, and suggest improvements. Output a well-formatted markdown report.", + "model": { + "provider": "zai-coding-plan", + "modelId": "glm-4.7", + "apiKey": "YOUR_ZAI_CODING_PLAN_API_KEY" + } + } + } + } + }, + { + "id": "pr-comment", + "type": "github.pr.comment", + "position": { "x": 400, "y": 740 }, + "data": { + "label": "Post Review Comment", + "config": { + "params": {}, + "inputOverrides": {} + } + } + } + ], + "edges": [ + { + "id": "edge-entry-installationid-to-clone", + "source": "entry-point", + "target": "clone-repo", + "sourceHandle": "installationId", + "targetHandle": "installationId" + }, + { + "id": "edge-entry-installationid-to-pr-context", + "source": "entry-point", + "target": "pr-context", + "sourceHandle": "installationId", + "targetHandle": "installationId" + }, + { + "id": "edge-entry-installationid-to-comment", + "source": "entry-point", + "target": "pr-comment", + "sourceHandle": "installationId", + "targetHandle": "installationId" + }, + { + "id": "edge-entry-owner-to-pr-context", + "source": "entry-point", + "target": "pr-context", + "sourceHandle": "owner", + "targetHandle": "owner" + }, + { + "id": "edge-entry-repo-to-pr-context", + "source": "entry-point", + "target": "pr-context", + "sourceHandle": "repo", + "targetHandle": "repo" + }, + { + "id": "edge-entry-pr-to-pr-context", + "source": "entry-point", + "target": "pr-context", + "sourceHandle": "prNumber", + "targetHandle": "pullNumber" + }, + { + "id": "edge-pr-context-to-agent-context", + "source": "pr-context", + "target": "opencode-agent", + "sourceHandle": "changedFilesDetailed", + "targetHandle": "context" + }, + { + "id": "edge-agent-to-comment", + "source": "opencode-agent", + "target": "pr-comment", + "sourceHandle": "report", + "targetHandle": "body" + }, + { + "id": "edge-entry-owner-to-comment", + "source": "entry-point", + "target": "pr-comment", + "sourceHandle": "owner", + "targetHandle": "owner" + }, + { + "id": "edge-entry-repo-to-comment", + "source": "entry-point", + "target": "pr-comment", + "sourceHandle": "repo", + "targetHandle": "repo" + }, + { + "id": "edge-entry-pr-to-comment", + "source": "entry-point", + "target": "pr-comment", + "sourceHandle": "prNumber", + "targetHandle": "prNumber" + } + ], + "viewport": { + "x": 0, + "y": 0, + "zoom": 0.65 + } +} diff --git a/docs/samples/07-opengrep-pr-trigger.json b/docs/samples/07-opengrep-pr-trigger.json new file mode 100644 index 000000000..e956d81eb --- /dev/null +++ b/docs/samples/07-opengrep-pr-trigger.json @@ -0,0 +1,13 @@ +{ + "name": "OpenGrep SAST on Pull Requests", + "description": "Automatically run OpenGrep SAST scanner when a pull request is opened, updated, or reopened. Posts findings as a PR comment and creates a GitHub Check Run.", + "repositoryPattern": "*", + "event": "pull_request", + "actions": ["opened", "synchronize", "reopened"], + "branches": [], + "workflowId": "0e842188-7a4a-45bb-a06a-7e5eecfa2151", + "postPrComment": true, + "createCheckRun": true, + "enabled": true, + "priority": 100 +} diff --git a/docs/samples/08-unified-ai-security-review.json b/docs/samples/08-unified-ai-security-review.json new file mode 100644 index 000000000..a505befc1 --- /dev/null +++ b/docs/samples/08-unified-ai-security-review.json @@ -0,0 +1,282 @@ +{ + "name": "Unified AI Security Review", + "description": "Run OpenGrep SAST and TruffleHog secret scans in parallel, feed raw findings + PR diffs directly to an OpenCode AI agent that produces a single unified PR comment.", + "nodes": [ + { + "id": "entry-point", + "type": "core.workflow.entrypoint", + "position": { "x": 400, "y": 0 }, + "data": { + "label": "Entry Point", + "config": { + "params": { + "runtimeInputs": [ + { + "id": "owner", + "label": "Owner", + "type": "text", + "required": true, + "description": "Repository owner (org or user)" + }, + { + "id": "repo", + "label": "Repo", + "type": "text", + "required": true, + "description": "Repository name" + }, + { + "id": "prNumber", + "label": "PR Number", + "type": "number", + "required": true, + "description": "Pull request number to review" + }, + { + "id": "repository", + "label": "Repository", + "type": "text", + "required": true, + "description": "Full repository in owner/repo format" + }, + { + "id": "installationId", + "label": "Installation ID", + "type": "number", + "required": false, + "description": "GitHub App installation ID (auto-injected by trigger)" + } + ] + }, + "inputOverrides": {} + } + } + }, + { + "id": "clone-repo", + "type": "github.repo.clone", + "position": { "x": 200, "y": 200 }, + "data": { + "label": "Clone Repository", + "config": { + "params": { + "depth": 0 + }, + "inputOverrides": {} + } + } + }, + { + "id": "pr-context", + "type": "github.pr.context", + "position": { "x": 600, "y": 200 }, + "data": { + "label": "Get PR Context", + "config": { + "params": {}, + "inputOverrides": {} + } + } + }, + { + "id": "opengrep-scanner", + "type": "scanner.opengrep", + "position": { "x": 100, "y": 440 }, + "data": { + "label": "OpenGrep SAST Scanner", + "config": { + "params": { + "ruleset": "auto", + "excludePaths": ["node_modules", "vendor", ".git", "dist", "build"], + "timeout": 300, + "jobs": 4 + }, + "inputOverrides": {} + } + } + }, + { + "id": "trufflehog-scanner", + "type": "scanner.trufflehog", + "position": { "x": 500, "y": 440 }, + "data": { + "label": "TruffleHog Secret Scanner", + "config": { + "params": { + "onlyVerified": false, + "maxDepth": 50, + "concurrency": 8 + }, + "inputOverrides": {} + } + } + }, + { + "id": "opencode-agent", + "type": "core.ai.opencode", + "position": { "x": 300, "y": 700 }, + "data": { + "label": "Unified AI Security & Code Review", + "config": { + "params": { + "systemPrompt": "You are ShipSec AI, a unified security and code review agent. You receive:\n\n1. **PR Code Diffs** in /workspace/context.json — files changed in this PR with filenames, status, additions, deletions, and unified diff patches.\n2. **OpenGrep SAST Results** in /workspace/supplementary-a.txt — raw Semgrep/OpenGrep JSON output with SAST findings (may be empty if no issues found).\n3. **TruffleHog Secret Scan Results** in /workspace/supplementary-b.txt — raw TruffleHog JSONL output with secret findings (may be empty if no secrets found).\n\nYour job:\n\n### A. Parse & Contextualize Scanner Findings\n- Parse the raw JSON/JSONL scanner output files.\n- For each finding, cross-reference with the PR diffs to assess true positive vs false positive.\n- Focus on findings in files changed by this PR.\n\n### B. Independent Code Review\n- Review all diffs for issues scanners miss: logic errors, race conditions, missing error handling, input validation gaps, API misuse, performance issues, readability.\n\n### C. Unified Report\n\nOutput ONLY markdown. No preamble, greetings, or debug output. Start directly:\n\n## \"ShipSec\" ShipSec Unified Security & Code Review\n\n### Executive Summary\n1-3 sentences: what this PR does and overall risk posture.\n\n### Findings Summary\n\n| | Severity | Category | File | Line | Finding |\n|---|----------|----------|------|------|---------|\n| \ud83d\udd34 | Critical | SAST/Secrets/Code Review | `file` | 42 | One-line description |\n| \ud83d\udfe0 | High | ... | ... | ... | ... |\n| \ud83d\udfe1 | Medium | ... | ... | ... | ... |\n| \ud83d\udd35 | Low | ... | ... | ... | ... |\n| \u2139\ufe0f | Info | ... | ... | ... | ... |\n\nSeverity icons: \ud83d\udd34 Critical, \ud83d\udfe0 High, \ud83d\udfe1 Medium, \ud83d\udd35 Low, \u2139\ufe0f Info\nCategory: **SAST** (OpenGrep), **Secrets** (TruffleHog), **Code Review** (your analysis)\nInclude ALL findings from all sources in one table, sorted by severity.\n\n### SAST Findings\nFor each OpenGrep finding: state true/false positive, explain risk, suggest fix.\nIf none: \"No SAST issues detected.\"\n\n### Secret Scan Findings\nFor each TruffleHog finding: assess test/mock vs real credential, recommend remediation.\nIf none: \"No exposed secrets detected.\"\n\n### Code Review Findings\nYour own findings not caught by scanners. For each: severity, file:line, issue, fix.\nIf none: \"No additional code quality issues identified.\"\n\n### Recommendations\nTop 3-5 prioritized action items.\n\n### Overall Assessment\nOne of:\n- \u2705 **Approve** \u2014 No critical/high issues, safe to merge\n- \u274c **Request Changes** \u2014 Critical or high-severity issues must be addressed\n- \ud83d\udcac **Comment** \u2014 Medium/low findings worth discussing\n\n---\n\"ShipSec\" ShipSec Unified Review \u2022 Powered by OpenGrep + TruffleHog + AI", + "autoApprove": true + }, + "inputOverrides": { + "task": "Review this pull request for security vulnerabilities, exposed secrets, and code quality issues. Use the scanner findings in the supplementary files and the PR diffs in context.json to produce a unified security and code review report.", + "model": { + "provider": "zai-coding-plan", + "modelId": "glm-4.7", + "apiKey": "YOUR_ZAI_CODING_PLAN_API_KEY" + } + } + } + } + }, + { + "id": "pr-comment", + "type": "github.pr.comment", + "position": { "x": 300, "y": 960 }, + "data": { + "label": "Post Unified Review Comment", + "config": { + "params": {}, + "inputOverrides": {} + } + } + } + ], + "edges": [ + { + "id": "edge-entry-repository-to-clone", + "source": "entry-point", + "target": "clone-repo", + "sourceHandle": "repository", + "targetHandle": "repository" + }, + { + "id": "edge-entry-installationid-to-clone", + "source": "entry-point", + "target": "clone-repo", + "sourceHandle": "installationId", + "targetHandle": "installationId" + }, + { + "id": "edge-entry-installationid-to-pr-context", + "source": "entry-point", + "target": "pr-context", + "sourceHandle": "installationId", + "targetHandle": "installationId" + }, + { + "id": "edge-entry-owner-to-pr-context", + "source": "entry-point", + "target": "pr-context", + "sourceHandle": "owner", + "targetHandle": "owner" + }, + { + "id": "edge-entry-repo-to-pr-context", + "source": "entry-point", + "target": "pr-context", + "sourceHandle": "repo", + "targetHandle": "repo" + }, + { + "id": "edge-entry-pr-to-pr-context", + "source": "entry-point", + "target": "pr-context", + "sourceHandle": "prNumber", + "targetHandle": "pullNumber" + }, + { + "id": "edge-clone-to-opengrep", + "source": "clone-repo", + "target": "opengrep-scanner", + "sourceHandle": "volumeName", + "targetHandle": "volumeName" + }, + { + "id": "edge-clone-to-trufflehog", + "source": "clone-repo", + "target": "trufflehog-scanner", + "sourceHandle": "volumeName", + "targetHandle": "volumeName" + }, + { + "id": "edge-pr-context-changedfiles-to-opengrep", + "source": "pr-context", + "target": "opengrep-scanner", + "sourceHandle": "changedFiles", + "targetHandle": "changedFiles" + }, + { + "id": "edge-pr-context-changedfiles-to-trufflehog", + "source": "pr-context", + "target": "trufflehog-scanner", + "sourceHandle": "changedFiles", + "targetHandle": "changedFiles" + }, + { + "id": "edge-opengrep-to-opencode", + "source": "opengrep-scanner", + "target": "opencode-agent", + "sourceHandle": "rawOutput", + "targetHandle": "supplementaryA" + }, + { + "id": "edge-trufflehog-to-opencode", + "source": "trufflehog-scanner", + "target": "opencode-agent", + "sourceHandle": "rawOutput", + "targetHandle": "supplementaryB" + }, + { + "id": "edge-pr-context-to-opencode", + "source": "pr-context", + "target": "opencode-agent", + "sourceHandle": "changedFilesDetailed", + "targetHandle": "context" + }, + { + "id": "edge-opencode-to-pr-comment", + "source": "opencode-agent", + "target": "pr-comment", + "sourceHandle": "report", + "targetHandle": "body" + }, + { + "id": "edge-entry-installationid-to-pr-comment", + "source": "entry-point", + "target": "pr-comment", + "sourceHandle": "installationId", + "targetHandle": "installationId" + }, + { + "id": "edge-entry-owner-to-pr-comment", + "source": "entry-point", + "target": "pr-comment", + "sourceHandle": "owner", + "targetHandle": "owner" + }, + { + "id": "edge-entry-repo-to-pr-comment", + "source": "entry-point", + "target": "pr-comment", + "sourceHandle": "repo", + "targetHandle": "repo" + }, + { + "id": "edge-entry-pr-to-pr-comment", + "source": "entry-point", + "target": "pr-comment", + "sourceHandle": "prNumber", + "targetHandle": "prNumber" + } + ], + "viewport": { + "x": 0, + "y": 0, + "zoom": 0.5 + } +} diff --git a/openapi.json b/openapi.json index 9b59c6ad1..c783d58cc 100644 --- a/openapi.json +++ b/openapi.json @@ -15,57 +15,6 @@ ] } }, - "/api/v1/auth/validate": { - "get": { - "operationId": "AppController_validateAuth", - "parameters": [], - "responses": { - "200": { - "description": "" - } - }, - "tags": [ - "App" - ] - } - }, - "/api/v1/auth/login": { - "post": { - "operationId": "AppController_login", - "parameters": [ - { - "name": "authorization", - "required": true, - "in": "header", - "schema": { - "type": "string" - } - } - ], - "responses": { - "201": { - "description": "" - } - }, - "tags": [ - "App" - ] - } - }, - "/api/v1/auth/logout": { - "post": { - "operationId": "AppController_logout", - "parameters": [], - "responses": { - "201": { - "description": "" - } - }, - "tags": [ - "App" - ] - } - }, "/api/v1/agents/{agentRunId}/parts": { "get": { "operationId": "AgentsController_parts", @@ -346,9 +295,6 @@ "workflowId": { "type": "string" }, - "organizationId": { - "type": "string" - }, "status": { "type": "string", "enum": [ @@ -467,9 +413,6 @@ "workflowId": { "type": "string" }, - "organizationId": { - "type": "string" - }, "status": { "type": "string", "enum": [ @@ -2513,147 +2456,6 @@ ] } }, - "/api/v1/analytics/query": { - "post": { - "operationId": "AnalyticsController_queryAnalytics", - "parameters": [ - { - "name": "X-RateLimit-Remaining", - "in": "header", - "description": "Number of requests remaining in the current time window", - "schema": { - "type": "integer", - "example": 99 - } - }, - { - "name": "X-RateLimit-Limit", - "in": "header", - "description": "Maximum number of requests allowed per minute", - "schema": { - "type": "integer", - "example": 100 - } - } - ], - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/AnalyticsQueryRequestDto" - } - } - } - }, - "responses": { - "200": { - "description": "Query analytics data for the authenticated organization", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/AnalyticsQueryResponseDto" - } - } - } - } - }, - "tags": [ - "analytics" - ] - } - }, - "/api/v1/analytics/settings": { - "get": { - "operationId": "AnalyticsController_getAnalyticsSettings", - "parameters": [], - "responses": { - "200": { - "description": "Get analytics settings for the authenticated organization", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/AnalyticsSettingsResponseDto" - } - } - } - } - }, - "tags": [ - "analytics" - ] - }, - "put": { - "operationId": "AnalyticsController_updateAnalyticsSettings", - "parameters": [], - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/UpdateAnalyticsSettingsDto" - } - } - } - }, - "responses": { - "200": { - "description": "Update analytics settings for the authenticated organization", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/AnalyticsSettingsResponseDto" - } - } - } - } - }, - "tags": [ - "analytics" - ] - } - }, - "/api/v1/analytics/ensure-tenant": { - "post": { - "operationId": "AnalyticsController_ensureTenant", - "parameters": [ - { - "name": "x-internal-token", - "required": true, - "in": "header", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "Ensure tenant resources exist for organization", - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "success": { - "type": "boolean" - }, - "securityEnabled": { - "type": "boolean" - }, - "message": { - "type": "string" - } - } - } - } - } - } - }, - "tags": [ - "analytics" - ] - } - }, "/api/v1/api-keys": { "get": { "operationId": "ApiKeysController_list", @@ -3205,23 +3007,20 @@ "type": "string" } }, - "toolProvider": { + "agentTool": { "type": "object", "nullable": true, "properties": { - "kind": { - "type": "string", - "enum": [ - "component", - "mcp-server", - "mcp-group" - ] + "enabled": { + "type": "boolean" }, - "name": { - "type": "string" + "toolName": { + "type": "string", + "nullable": true }, - "description": { - "type": "string" + "toolDescription": { + "type": "string", + "nullable": true } } } @@ -5437,17 +5236,7 @@ "parameters": [], "responses": { "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "array", - "items": { - "$ref": "#/components/schemas/GroupTemplateDto" - } - } - } - } + "description": "" } }, "summary": "List available MCP group templates", @@ -5944,16 +5733,40 @@ ] } }, - "/api/v1/internal/mcp/register-mcp-server": { + "/api/v1/internal/mcp/register-remote": { + "post": { + "operationId": "InternalMcpController_registerRemote", + "parameters": [], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RegisterRemoteMcpInput" + } + } + } + }, + "responses": { + "201": { + "description": "" + } + }, + "tags": [ + "InternalMcp" + ] + } + }, + "/api/v1/internal/mcp/register-local": { "post": { - "operationId": "InternalMcpController_registerMcpServer", + "operationId": "InternalMcpController_registerLocal", "parameters": [], "requestBody": { "required": true, "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/RegisterMcpServerInput" + "$ref": "#/components/schemas/RegisterLocalMcpInput" } } } @@ -6146,63 +5959,23 @@ ] } }, - "/api/v1/audit-logs": { + "/api/v1/github/status": { "get": { - "operationId": "AuditLogsController_list", + "operationId": "GitHubAppController_getStatus", "parameters": [], "responses": { "200": { - "description": "List audit log events for the authenticated organization", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ListAuditLogsResponseDto" - } - } - } + "description": "" } }, "tags": [ - "audit-logs" + "GitHubApp" ] } }, - "/api/v1/testing/webhooks": { - "post": { - "operationId": "TestingWebhookController_acceptWebhook", - "parameters": [ - { - "name": "status", - "required": false, - "in": "query", - "schema": { - "minimum": 100, - "maximum": 599, - "type": "integer" - } - }, - { - "name": "delayMs", - "required": false, - "in": "query", - "schema": { - "minimum": 0, - "maximum": 60000, - "type": "integer" - } - } - ], - "responses": { - "201": { - "description": "" - } - }, - "tags": [ - "TestingWebhook" - ] - }, + "/api/v1/github/installations": { "get": { - "operationId": "TestingWebhookController_listRecords", + "operationId": "GitHubAppController_listInstallations", "parameters": [], "responses": { "200": { @@ -6210,11 +5983,13 @@ } }, "tags": [ - "TestingWebhook" + "GitHubApp" ] - }, - "delete": { - "operationId": "TestingWebhookController_clearRecords", + } + }, + "/api/v1/github/repos": { + "get": { + "operationId": "GitHubAppController_listRepositories", "parameters": [], "responses": { "200": { @@ -6222,14 +5997,440 @@ } }, "tags": [ - "TestingWebhook" + "GitHubApp" ] } }, - "/api/v1/testing/webhooks/latest": { + "/api/v1/github/repos/{id}": { "get": { - "operationId": "TestingWebhookController_latestRecord", - "parameters": [], + "operationId": "GitHubAppController_getRepository", + "parameters": [ + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "" + } + }, + "tags": [ + "GitHubApp" + ] + } + }, + "/api/v1/github/repos/{id}/branches": { + "get": { + "operationId": "GitHubAppController_listBranches", + "parameters": [ + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "" + } + }, + "tags": [ + "GitHubApp" + ] + } + }, + "/api/v1/github/repos/{id}/scans": { + "patch": { + "operationId": "GitHubAppController_toggleRepoScans", + "parameters": [ + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "" + } + }, + "tags": [ + "GitHubApp" + ] + } + }, + "/api/v1/github/installations/setup": { + "post": { + "operationId": "GitHubAppController_setupInstallation", + "parameters": [], + "responses": { + "201": { + "description": "" + } + }, + "tags": [ + "GitHubApp" + ] + } + }, + "/api/v1/github/installations/{installationId}/token": { + "post": { + "operationId": "GitHubAppController_getInstallationToken", + "parameters": [ + { + "name": "installationId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "201": { + "description": "" + } + }, + "tags": [ + "GitHubApp" + ] + } + }, + "/api/v1/github/repos/{id}/scan": { + "post": { + "operationId": "GitHubAppController_triggerScan", + "parameters": [ + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "201": { + "description": "" + } + }, + "tags": [ + "GitHubApp" + ] + } + }, + "/api/v1/github/repos/sync": { + "post": { + "operationId": "GitHubAppController_syncRepositories", + "parameters": [], + "responses": { + "201": { + "description": "" + } + }, + "tags": [ + "GitHubApp" + ] + } + }, + "/api/v1/github/triggers": { + "get": { + "operationId": "GitHubAppController_listTriggerRules", + "parameters": [], + "responses": { + "200": { + "description": "" + } + }, + "tags": [ + "GitHubApp" + ] + }, + "post": { + "operationId": "GitHubAppController_createTriggerRule", + "parameters": [], + "responses": { + "201": { + "description": "" + } + }, + "tags": [ + "GitHubApp" + ] + } + }, + "/api/v1/github/triggers/{id}": { + "get": { + "operationId": "GitHubAppController_getTriggerRule", + "parameters": [ + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "" + } + }, + "tags": [ + "GitHubApp" + ] + }, + "patch": { + "operationId": "GitHubAppController_updateTriggerRule", + "parameters": [ + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "" + } + }, + "tags": [ + "GitHubApp" + ] + }, + "delete": { + "operationId": "GitHubAppController_deleteTriggerRule", + "parameters": [ + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "204": { + "description": "" + } + }, + "tags": [ + "GitHubApp" + ] + } + }, + "/api/v1/github/scans": { + "get": { + "operationId": "GitHubAppController_listScanResults", + "parameters": [ + { + "name": "repositoryId", + "required": true, + "in": "query", + "schema": { + "type": "string" + } + }, + { + "name": "status", + "required": true, + "in": "query", + "schema": { + "type": "string" + } + }, + { + "name": "source", + "required": true, + "in": "query", + "schema": { + "type": "string" + } + }, + { + "name": "scheduleId", + "required": true, + "in": "query", + "schema": { + "type": "string" + } + }, + { + "name": "triggerRuleId", + "required": true, + "in": "query", + "schema": { + "type": "string" + } + }, + { + "name": "limit", + "required": true, + "in": "query", + "schema": { + "type": "string" + } + }, + { + "name": "offset", + "required": true, + "in": "query", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "" + } + }, + "tags": [ + "GitHubApp" + ] + } + }, + "/api/v1/github/scans/{id}": { + "get": { + "operationId": "GitHubAppController_getScanResult", + "parameters": [ + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "" + } + }, + "tags": [ + "GitHubApp" + ] + } + }, + "/api/v1/github/webhook": { + "post": { + "operationId": "GitHubAppController_handleWebhook", + "parameters": [ + { + "name": "x-github-event", + "required": true, + "in": "header", + "schema": { + "type": "string" + } + }, + { + "name": "x-hub-signature-256", + "required": true, + "in": "header", + "schema": { + "type": "string" + } + }, + { + "name": "x-github-delivery", + "required": true, + "in": "header", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "" + } + }, + "tags": [ + "GitHubApp" + ] + } + }, + "/api/v1/testing/webhooks": { + "post": { + "operationId": "TestingWebhookController_acceptWebhook", + "parameters": [ + { + "name": "status", + "required": false, + "in": "query", + "schema": { + "minimum": 100, + "maximum": 599, + "type": "integer" + } + }, + { + "name": "delayMs", + "required": false, + "in": "query", + "schema": { + "minimum": 0, + "maximum": 60000, + "type": "integer" + } + } + ], + "responses": { + "201": { + "description": "" + } + }, + "tags": [ + "TestingWebhook" + ] + }, + "get": { + "operationId": "TestingWebhookController_listRecords", + "parameters": [], + "responses": { + "200": { + "description": "" + } + }, + "tags": [ + "TestingWebhook" + ] + }, + "delete": { + "operationId": "TestingWebhookController_clearRecords", + "parameters": [], + "responses": { + "200": { + "description": "" + } + }, + "tags": [ + "TestingWebhook" + ] + } + }, + "/api/v1/testing/webhooks/latest": { + "get": { + "operationId": "TestingWebhookController_latestRecord", + "parameters": [], "responses": { "200": { "description": "" @@ -7855,166 +8056,31 @@ "type": "string", "nullable": true }, - "createdAt": { - "type": "string", - "format": "date-time", - "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d(?::[0-5]\\d(?:\\.\\d+)?)?(?:Z))$" - } - }, - "required": [ - "id", - "runId", - "workflowId", - "componentRef", - "fileId", - "name", - "mimeType", - "size", - "destinations", - "createdAt" - ] - } - } - }, - "required": [ - "artifacts" - ] - }, - "AnalyticsQueryRequestDto": { - "type": "object", - "properties": { - "query": { - "type": "object", - "description": "OpenSearch DSL query object", - "example": { - "match_all": {} - } - }, - "size": { - "type": "number", - "description": "Number of results to return", - "example": 10, - "default": 10, - "minimum": 0, - "maximum": 1000 - }, - "from": { - "type": "number", - "description": "Offset for pagination", - "example": 0, - "default": 0, - "minimum": 0, - "maximum": 10000 - }, - "aggs": { - "type": "object", - "description": "OpenSearch aggregations object", - "example": { - "components": { - "terms": { - "field": "component_id" - } - } - } - } - } - }, - "AnalyticsQueryResponseDto": { - "type": "object", - "properties": { - "total": { - "type": "number", - "description": "Total number of matching documents", - "example": 100 - }, - "hits": { - "type": "array", - "description": "Search hits", - "items": { - "type": "object" - } - }, - "aggregations": { - "type": "object", - "description": "Aggregation results" - } - }, - "required": [ - "total", - "hits" - ] - }, - "AnalyticsSettingsResponseDto": { - "type": "object", - "properties": { - "organizationId": { - "type": "string", - "description": "Organization ID", - "example": "org_abc123" - }, - "subscriptionTier": { - "type": "string", - "description": "Subscription tier", - "enum": [ - "free", - "pro", - "enterprise" - ], - "example": "free" - }, - "analyticsRetentionDays": { - "type": "number", - "description": "Data retention period in days", - "example": 30 - }, - "maxRetentionDays": { - "type": "number", - "description": "Maximum retention days allowed for this tier", - "example": 30 - }, - "createdAt": { - "format": "date-time", - "type": "string", - "description": "Timestamp when settings were created", - "example": "2026-01-20T00:00:00.000Z" - }, - "updatedAt": { - "format": "date-time", - "type": "string", - "description": "Timestamp when settings were last updated", - "example": "2026-01-20T00:00:00.000Z" + "createdAt": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d(?::[0-5]\\d(?:\\.\\d+)?)?(?:Z))$" + } + }, + "required": [ + "id", + "runId", + "workflowId", + "componentRef", + "fileId", + "name", + "mimeType", + "size", + "destinations", + "createdAt" + ] + } } }, "required": [ - "organizationId", - "subscriptionTier", - "analyticsRetentionDays", - "maxRetentionDays", - "createdAt", - "updatedAt" + "artifacts" ] }, - "UpdateAnalyticsSettingsDto": { - "type": "object", - "properties": { - "analyticsRetentionDays": { - "type": "number", - "description": "Data retention period in days (must be within tier limits)", - "example": 30, - "minimum": 1, - "maximum": 365 - }, - "subscriptionTier": { - "type": "string", - "description": "Subscription tier (optional - usually set by billing system)", - "enum": [ - "free", - "pro", - "enterprise" - ] - } - } - }, "ApiKeyResponseDto": { "type": "object", "properties": { @@ -8070,23 +8136,11 @@ "read", "cancel" ] - }, - "audit": { - "type": "object", - "properties": { - "read": { - "type": "boolean" - } - }, - "required": [ - "read" - ] } }, "required": [ "workflows", - "runs", - "audit" + "runs" ] }, "isActive": { @@ -8180,23 +8234,11 @@ "read", "cancel" ] - }, - "audit": { - "type": "object", - "properties": { - "read": { - "type": "boolean" - } - }, - "required": [ - "read" - ] } }, "required": [ "workflows", - "runs", - "audit" + "runs" ] }, "expiresAt": { @@ -8273,23 +8315,11 @@ "read", "cancel" ] - }, - "audit": { - "type": "object", - "properties": { - "read": { - "type": "boolean" - } - }, - "required": [ - "read" - ] } }, "required": [ "workflows", - "runs", - "audit" + "runs" ] }, "isActive": { @@ -8388,23 +8418,11 @@ "read", "cancel" ] - }, - "audit": { - "type": "object", - "properties": { - "read": { - "type": "boolean" - } - }, - "required": [ - "read" - ] } }, "required": [ "workflows", - "runs", - "audit" + "runs" ] }, "isActive": { @@ -10088,9 +10106,7 @@ "propertyNames": { "type": "string" }, - "additionalProperties": { - "type": "string" - }, + "additionalProperties": {}, "nullable": true }, "defaultDockerImage": { @@ -10128,118 +10144,6 @@ "updatedAt" ] }, - "GroupTemplateDto": { - "type": "object", - "properties": { - "slug": { - "type": "string", - "minLength": 1 - }, - "name": { - "type": "string", - "minLength": 1 - }, - "description": { - "type": "string" - }, - "credentialContractName": { - "type": "string", - "minLength": 1 - }, - "credentialMapping": { - "type": "object", - "propertyNames": { - "type": "string" - }, - "additionalProperties": { - "type": "string" - } - }, - "defaultDockerImage": { - "type": "string", - "minLength": 1 - }, - "version": { - "type": "object", - "properties": { - "major": { - "type": "number" - }, - "minor": { - "type": "number" - }, - "patch": { - "type": "number" - } - }, - "required": [ - "major", - "minor", - "patch" - ] - }, - "servers": { - "type": "array", - "items": { - "type": "object", - "properties": { - "name": { - "type": "string", - "minLength": 1 - }, - "description": { - "type": "string" - }, - "transportType": { - "type": "string", - "enum": [ - "http", - "stdio", - "sse", - "websocket" - ] - }, - "endpoint": { - "type": "string" - }, - "command": { - "type": "string" - }, - "args": { - "type": "array", - "items": { - "type": "string" - } - }, - "recommended": { - "type": "boolean" - }, - "defaultSelected": { - "type": "boolean" - } - }, - "required": [ - "name", - "transportType", - "recommended", - "defaultSelected" - ] - } - }, - "templateHash": { - "type": "string" - } - }, - "required": [ - "slug", - "name", - "credentialContractName", - "defaultDockerImage", - "version", - "servers", - "templateHash" - ] - }, "CreateMcpGroupDto": { "type": "object", "properties": { @@ -10264,9 +10168,7 @@ "propertyNames": { "type": "string" }, - "additionalProperties": { - "type": "string" - }, + "additionalProperties": {}, "nullable": true }, "defaultDockerImage": { @@ -10303,9 +10205,7 @@ "propertyNames": { "type": "string" }, - "additionalProperties": { - "type": "string" - }, + "additionalProperties": {}, "nullable": true }, "defaultDockerImage": { @@ -10499,9 +10399,7 @@ "propertyNames": { "type": "string" }, - "additionalProperties": { - "type": "string" - }, + "additionalProperties": {}, "nullable": true }, "defaultDockerImage": { @@ -10549,7 +10447,11 @@ "type": "object", "properties": {} }, - "RegisterMcpServerInput": { + "RegisterRemoteMcpInput": { + "type": "object", + "properties": {} + }, + "RegisterLocalMcpInput": { "type": "object", "properties": {} }, @@ -10902,111 +10804,6 @@ "workflowId", "status" ] - }, - "ListAuditLogsResponseDto": { - "type": "object", - "properties": { - "items": { - "type": "array", - "items": { - "type": "object", - "properties": { - "id": { - "type": "string", - "format": "uuid", - "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" - }, - "organizationId": { - "type": "string", - "nullable": true - }, - "actorId": { - "type": "string", - "nullable": true - }, - "actorType": { - "type": "string", - "enum": [ - "user", - "api-key", - "internal", - "unknown" - ] - }, - "actorDisplay": { - "type": "string", - "nullable": true - }, - "action": { - "type": "string" - }, - "resourceType": { - "type": "string", - "enum": [ - "workflow", - "secret", - "api_key", - "webhook", - "artifact", - "analytics" - ] - }, - "resourceId": { - "type": "string", - "nullable": true - }, - "resourceName": { - "type": "string", - "nullable": true - }, - "metadata": { - "type": "object", - "propertyNames": { - "type": "string" - }, - "additionalProperties": {}, - "nullable": true - }, - "ip": { - "type": "string", - "nullable": true - }, - "userAgent": { - "type": "string", - "nullable": true - }, - "createdAt": { - "type": "string", - "format": "date-time", - "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d(?::[0-5]\\d(?:\\.\\d+)?)?(?:Z))$" - } - }, - "required": [ - "id", - "organizationId", - "actorId", - "actorType", - "actorDisplay", - "action", - "resourceType", - "resourceId", - "resourceName", - "metadata", - "ip", - "userAgent", - "createdAt" - ] - } - }, - "nextCursor": { - "type": "string", - "nullable": true - } - }, - "required": [ - "items", - "nextCursor" - ] } } } diff --git a/packages/backend-client/src/client.ts b/packages/backend-client/src/client.ts index 6886c66a8..54d7d7560 100644 --- a/packages/backend-client/src/client.ts +++ b/packages/backend-client/src/client.ts @@ -1879,14 +1879,14 @@ export interface paths { patch?: never; trace?: never; }; - "/api/v1/audit-logs": { + "/api/v1/github/status": { parameters: { query?: never; header?: never; path?: never; cookie?: never; }; - get: operations["AuditLogsController_list"]; + get: operations["GitHubAppController_getStatus"]; put?: never; post?: never; delete?: never; @@ -1895,6 +1895,230 @@ export interface paths { patch?: never; trace?: never; }; + "/api/v1/github/installations": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["GitHubAppController_listInstallations"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/v1/github/repos": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["GitHubAppController_listRepositories"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/v1/github/repos/{id}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["GitHubAppController_getRepository"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/v1/github/repos/{id}/branches": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["GitHubAppController_listBranches"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/v1/github/repos/{id}/scans": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch: operations["GitHubAppController_toggleRepoScans"]; + trace?: never; + }; + "/api/v1/github/installations/setup": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: operations["GitHubAppController_setupInstallation"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/v1/github/installations/{installationId}/token": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: operations["GitHubAppController_getInstallationToken"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/v1/github/repos/{id}/scan": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: operations["GitHubAppController_triggerScan"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/v1/github/repos/sync": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: operations["GitHubAppController_syncRepositories"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/v1/github/triggers": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["GitHubAppController_listTriggerRules"]; + put?: never; + post: operations["GitHubAppController_createTriggerRule"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/v1/github/triggers/{id}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["GitHubAppController_getTriggerRule"]; + put?: never; + post?: never; + delete: operations["GitHubAppController_deleteTriggerRule"]; + options?: never; + head?: never; + patch: operations["GitHubAppController_updateTriggerRule"]; + trace?: never; + }; + "/api/v1/github/scans": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["GitHubAppController_listScanResults"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/v1/github/scans/{id}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["GitHubAppController_getScanResult"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/v1/github/webhook": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: operations["GitHubAppController_handleWebhook"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/v1/testing/webhooks": { parameters: { query?: never; @@ -2539,9 +2763,6 @@ export interface components { read: boolean; cancel: boolean; }; - audit: { - read: boolean; - }; }; isActive: boolean; /** Format: date-time */ @@ -2567,9 +2788,6 @@ export interface components { read: boolean; cancel: boolean; }; - audit: { - read: boolean; - }; }; /** Format: date-time */ expiresAt?: string; @@ -2592,9 +2810,6 @@ export interface components { read: boolean; cancel: boolean; }; - audit: { - read: boolean; - }; }; isActive: boolean; /** Format: date-time */ @@ -2622,9 +2837,6 @@ export interface components { read: boolean; cancel: boolean; }; - audit: { - read: boolean; - }; }; isActive?: boolean; rateLimit?: number | null; @@ -3411,30 +3623,6 @@ export interface components { error?: string; errorCode?: string; }; - ListAuditLogsResponseDto: { - items: { - /** Format: uuid */ - id: string; - organizationId: string | null; - actorId: string | null; - /** @enum {string} */ - actorType: "user" | "api-key" | "internal" | "unknown"; - actorDisplay: string | null; - action: string; - /** @enum {string} */ - resourceType: "workflow" | "secret" | "api_key" | "webhook" | "artifact" | "analytics"; - resourceId: string | null; - resourceName: string | null; - metadata: { - [key: string]: unknown; - } | null; - ip: string | null; - userAgent: string | null; - /** Format: date-time */ - createdAt: string; - }[]; - nextCursor: string | null; - }; }; responses: never; parameters: never; @@ -7139,7 +7327,7 @@ export interface operations { }; }; }; - AuditLogsController_list: { + GitHubAppController_getStatus: { parameters: { query?: never; header?: never; @@ -7148,14 +7336,330 @@ export interface operations { }; requestBody?: never; responses: { - /** @description List audit log events for the authenticated organization */ 200: { headers: { [name: string]: unknown; }; - content: { - "application/json": components["schemas"]["ListAuditLogsResponseDto"]; + content?: never; + }; + }; + }; + GitHubAppController_listInstallations: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + GitHubAppController_listRepositories: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + GitHubAppController_getRepository: { + parameters: { + query?: never; + header?: never; + path: { + id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + GitHubAppController_listBranches: { + parameters: { + query?: never; + header?: never; + path: { + id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + GitHubAppController_toggleRepoScans: { + parameters: { + query?: never; + header?: never; + path: { + id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + GitHubAppController_setupInstallation: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + 201: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + GitHubAppController_getInstallationToken: { + parameters: { + query?: never; + header?: never; + path: { + installationId: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 201: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + GitHubAppController_triggerScan: { + parameters: { + query?: never; + header?: never; + path: { + id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 201: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + GitHubAppController_syncRepositories: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + 201: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + GitHubAppController_listTriggerRules: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + GitHubAppController_createTriggerRule: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + 201: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + GitHubAppController_getTriggerRule: { + parameters: { + query?: never; + header?: never; + path: { + id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + GitHubAppController_deleteTriggerRule: { + parameters: { + query?: never; + header?: never; + path: { + id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 204: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + GitHubAppController_updateTriggerRule: { + parameters: { + query?: never; + header?: never; + path: { + id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + GitHubAppController_listScanResults: { + parameters: { + query: { + repositoryId: string; + status: string; + source: string; + scheduleId: string; + triggerRuleId: string; + limit: string; + offset: string; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; }; + content?: never; + }; + }; + }; + GitHubAppController_getScanResult: { + parameters: { + query?: never; + header?: never; + path: { + id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + GitHubAppController_handleWebhook: { + parameters: { + query?: never; + header: { + "x-github-event": string; + "x-hub-signature-256": string; + "x-github-delivery": string; + }; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content?: never; }; }; }; From f2e52209925b372dd3835cce62996d1e26230425 Mon Sep 17 00:00:00 2001 From: betterclever Date: Mon, 20 Apr 2026 10:40:07 -0700 Subject: [PATCH 016/690] feat(worker): add GitHub and scanner components with security finding normalization --- packages/component-sdk/src/types.ts | 99 ++- worker/src/components/ai/opencode.ts | 113 +++- .../github/__tests__/post-pr-comment.test.ts | 226 +++++++ worker/src/components/github/clone-repo.ts | 480 ++++++++++++++ .../src/components/github/create-check-run.ts | 209 ++++++ worker/src/components/github/github-auth.ts | 114 ++++ .../src/components/github/post-pr-comment.ts | 226 +++++++ worker/src/components/github/pr-context.ts | 328 ++++++++++ .../src/components/github/update-check-run.ts | 200 ++++++ .../src/components/github/volume-cleanup.ts | 190 ++++++ worker/src/components/index.ts | 26 + .../scanners/__tests__/trufflehog.test.ts | 185 ++++++ worker/src/components/scanners/dependency.ts | 380 +++++++++++ worker/src/components/scanners/opengrep.ts | 465 +++++++++++++ worker/src/components/scanners/trivy.ts | 484 ++++++++++++++ worker/src/components/scanners/trufflehog.ts | 488 ++++++++++++++ .../findings-normalize-filter.test.ts | 49 ++ .../security/__tests__/trufflehog.test.ts | 236 ++++++- .../components/security/findings-markdown.ts | 332 ++++++++++ .../components/security/findings-normalize.ts | 611 ++++++++++++++++++ worker/src/components/security/trufflehog.ts | 50 +- 21 files changed, 5434 insertions(+), 57 deletions(-) create mode 100644 worker/src/components/github/__tests__/post-pr-comment.test.ts create mode 100644 worker/src/components/github/clone-repo.ts create mode 100644 worker/src/components/github/create-check-run.ts create mode 100644 worker/src/components/github/github-auth.ts create mode 100644 worker/src/components/github/post-pr-comment.ts create mode 100644 worker/src/components/github/pr-context.ts create mode 100644 worker/src/components/github/update-check-run.ts create mode 100644 worker/src/components/github/volume-cleanup.ts create mode 100644 worker/src/components/scanners/__tests__/trufflehog.test.ts create mode 100644 worker/src/components/scanners/dependency.ts create mode 100644 worker/src/components/scanners/opengrep.ts create mode 100644 worker/src/components/scanners/trivy.ts create mode 100644 worker/src/components/scanners/trufflehog.ts create mode 100644 worker/src/components/security/__tests__/findings-normalize-filter.test.ts create mode 100644 worker/src/components/security/findings-markdown.ts create mode 100644 worker/src/components/security/findings-normalize.ts diff --git a/packages/component-sdk/src/types.ts b/packages/component-sdk/src/types.ts index 13055a514..366f17319 100644 --- a/packages/component-sdk/src/types.ts +++ b/packages/component-sdk/src/types.ts @@ -14,7 +14,7 @@ import type { HttpInstrumentationOptions, HttpRequestInput } from './http/types' export type { ExecutionContextMetadata } from './interfaces'; -export type RunnerKind = 'inline' | 'docker' | 'remote' | 'k8s'; +export type RunnerKind = 'inline' | 'docker' | 'remote'; export interface InlineRunnerConfig { kind: 'inline'; @@ -90,6 +90,65 @@ export interface LogEventInput { metadata?: ExecutionContextMetadata; } +export interface McpServerSpec { + id: string; + name: string; + command: string; + args?: string[]; +} + +export type ToolProviderKind = + | 'component' // Component exposes itself as a tool + | 'mcp-server' // Component runs a single MCP server + | 'mcp-group'; // Component manages multiple MCP servers + +export interface ToolProviderConfig { + kind: ToolProviderKind; + + /** + * Tool name exposed to the agent. + * For 'component' kind, this is the tool name. + * For 'mcp-group', this is used as a prefix for child tools if needed. + */ + name: string; + + /** + * Description of what the tool(s) do, shown to the agent. + */ + description: string; + + /** + * Configuration for MCP-based tool providers. + * Required for 'mcp-server' and 'mcp-group' kinds. + */ + mcp?: { + /** Docker image to use for the MCP server(s) */ + image?: string; + /** Command to run if image is used (for 'mcp-server') */ + command?: string[]; + /** Mapping of environment variables to component inputs/params */ + credentialMapping?: Record; + /** Specification for individual servers in a group (for 'mcp-group') */ + servers?: McpServerSpec[]; + }; + + /** + * For 'component' kind, optional override for tool input schema. + * If not provided, it's inferred from component inputs. + */ + inputSchema?: any; + + /** + * Optional Docker configuration for 'component' kind tools that run via Docker + * but aren't full MCP servers (e.g., standard scanners). + */ + docker?: { + image: string; + command: string[]; + args?: string[]; + }; +} + export interface AgentTracePart { type: string; [key: string]: unknown; @@ -271,7 +330,8 @@ export type ComponentParameterType = | 'artifact' | 'variable-list' | 'form-fields' - | 'selection-options'; + | 'selection-options' + | 'tags'; export interface ComponentParameterOption { label: string; @@ -311,6 +371,7 @@ export type ComponentCategory = | 'ai' | 'mcp' | 'security' + | 'scanners' | 'it_ops' | 'notification' | 'manual_action' @@ -323,24 +384,6 @@ export type ComponentUiType = | 'process' | 'output'; -/** - * Configuration for exposing a component as an agent-callable tool. - */ -export interface AgentToolConfig { - /** Whether this component can be used as an agent tool */ - enabled: boolean; - /** - * Tool name exposed to the agent. Defaults to component slug with underscores. - * Should be descriptive and follow snake_case convention. - * @example 'check_ip_reputation', 'query_cloudtrail' - */ - toolName?: string; - /** - * Description of what the tool does, shown to the agent. - * Should clearly explain the tool's purpose and when to use it. - */ - toolDescription?: string; -} export interface ComponentUiMetadata { slug: string; @@ -359,12 +402,6 @@ export interface ComponentUiMetadata { examples?: string[]; /** UI-only component - should not be included in workflow execution */ uiOnly?: boolean; - /** - * Configuration for exposing this component as an agent-callable tool. - * When enabled, the component can be used in tool mode within workflows, - * allowing AI agents to invoke it via the MCP gateway. - */ - agentTool?: AgentToolConfig; } export interface ExecutionContext { @@ -377,6 +414,11 @@ export interface ExecutionContext { metadata: ExecutionContextMetadata; agentTracePublisher?: AgentTracePublisher; + // Workflow context (optional, available when running in workflow) + workflowId?: string; + workflowName?: string; + organizationId?: string | null; + // Service interfaces - implemented by adapters storage?: IFileStorageService; secrets?: ISecretsService; @@ -486,6 +528,11 @@ export interface ComponentDefinition< ui?: ComponentUiMetadata; requiresSecrets?: boolean; + /** + * Configuration for exposing this component (or its children) as agent-callable tools. + */ + toolProvider?: ToolProviderConfig; + /** Retry policy for this component (optional, uses default if not specified) */ retryPolicy?: ComponentRetryPolicy; diff --git a/worker/src/components/ai/opencode.ts b/worker/src/components/ai/opencode.ts index e8b9fd2e9..ae2c4260b 100644 --- a/worker/src/components/ai/opencode.ts +++ b/worker/src/components/ai/opencode.ts @@ -11,7 +11,7 @@ import { param, } from '@shipsec/component-sdk'; import { LLMProviderSchema, llmProviderContractName } from '@shipsec/contracts'; -import { createIsolatedVolume } from '../../utils/isolated-volume'; +import { IsolatedContainerVolume } from '../../utils/isolated-volume'; import { DEFAULT_GATEWAY_URL, getGatewaySessionToken } from './utils'; const inputSchema = inputs({ @@ -52,6 +52,32 @@ const inputSchema = inputs({ reason: 'Tool-mode port acts as a graph anchor; payloads are not consumed directly.', connectionType: { kind: 'contract', name: 'mcp.tool' }, }), + supplementaryA: port( + z + .string() + .optional() + .describe('Additional data written to /workspace/supplementary-a.txt for the agent.'), + { + label: 'Supplementary Input A', + description: + 'Optional text/data (e.g. scanner findings) written to a workspace file the agent can read.', + allowAny: true, + reason: 'Accepts any stringifiable data for the agent workspace.', + }, + ), + supplementaryB: port( + z + .string() + .optional() + .describe('Additional data written to /workspace/supplementary-b.txt for the agent.'), + { + label: 'Supplementary Input B', + description: + 'Optional text/data (e.g. scanner findings) written to a workspace file the agent can read.', + allowAny: true, + reason: 'Accepts any stringifiable data for the agent workspace.', + }, + ), }); const parameterSchema = parameters({ @@ -99,7 +125,7 @@ const definition = defineComponent({ category: 'ai', runner: { kind: 'docker', - image: 'ghcr.io/shipsecai/opencode:1.1.53', + image: 'ghcr.io/shipsecai/opencode:latest', entrypoint: 'opencode', // We will override this in execution network: 'host' as const, // Required to access localhost gateway command: ['help'], @@ -128,7 +154,7 @@ const definition = defineComponent({ }, }, async execute({ inputs, params }, context) { - const { task, context: taskContext, model } = inputs; + const { task, context: taskContext, model, supplementaryA, supplementaryB } = inputs; const { systemPrompt, providerConfig } = params; const { connectedToolNodeIds, organizationId } = context.metadata; @@ -234,17 +260,24 @@ Please investigate the issue and generate a detailed report. finalPrompt = defaultPrompt.replace('{{TASK}}', task); } - // Ask the agent to list available MCP tools first (best-effort). - finalPrompt = - `${finalPrompt}\n\n# MCP Tools\n` + - `Before you start, list the MCP tools you can see. If none are available, say so explicitly.`; + // Append supplementary file references to the prompt + const suppFiles: string[] = []; + if (supplementaryA) suppFiles.push('/workspace/supplementary-a.txt'); + if (supplementaryB) suppFiles.push('/workspace/supplementary-b.txt'); + if (suppFiles.length > 0) { + finalPrompt += `\n\n# Additional Data Files\n${suppFiles.map((f) => `- ${f}`).join('\n')}`; + } // 4. Setup Isolated Volume const tenantId = (context as any).tenantId ?? 'default-tenant'; - const volume = createIsolatedVolume(tenantId, context.runId); + const volume = new IsolatedContainerVolume(tenantId, context.runId); try { // 5. Execute Docker Container + // HACK: Fail fast after listing tools for faster iteration on MCP tool registration + // TODO: Remove this hack once MCP tool registration is working correctly + const HACK_FAIL_FAST_AFTER_TOOL_LIST = 'false'; + // Write a wrapper script to properly execute opencode with file reading // The script runs inside the container, so $(cat /workspace/prompt.txt) works correctly // Note: --quiet flag doesn't exist in opencode 1.1.34, use --log-level ERROR instead @@ -252,23 +285,26 @@ Please investigate the issue and generate a detailed report. '#!/bin/sh', 'set -e', 'cd /workspace', - 'echo "[OpenCode] Listing MCP tools before run..."', - 'opencode mcp list --log-level ERROR || true', - 'echo "[OpenCode] Starting agent run..."', + '# List MCP tools for debugging (stderr only)', + 'opencode mcp list --log-level ERROR >&2 || true', + '# Run the agent — only agent output goes to stdout', 'opencode run --log-level ERROR "$(cat /workspace/prompt.txt)"', '', ].join('\n'); const opencodeConfigJson = JSON.stringify(opencodeConfig, null, 2); - // Initialize workspace with config, context, prompt, and wrapper script - await volume.initialize({ + // Initialize workspace with config, context, prompt, wrapper script, and supplementary files + const workspaceFiles: Record = { 'context.json': contextJson, // Opencode prefers opencode.jsonc in cwd; keep opencode.json for backwards compat. 'opencode.jsonc': opencodeConfigJson, 'opencode.json': opencodeConfigJson, 'prompt.txt': finalPrompt, 'run.sh': wrapperScript, - }); + }; + if (supplementaryA) workspaceFiles['supplementary-a.txt'] = supplementaryA; + if (supplementaryB) workspaceFiles['supplementary-b.txt'] = supplementaryB; + await volume.initialize(workspaceFiles); const runnerConfig = { ...definition.runner, @@ -334,7 +370,7 @@ Please investigate the issue and generate a detailed report. } return outputSchema.parse({ - report: stdout, // The markdown report is expected in stdout + report: cleanReport(stdout), rawOutput: `STDOUT:\n${stdout}\n\nSTDERR:\n${stderr}`, }); } finally { @@ -347,6 +383,53 @@ function isRecord(value: unknown): value is Record { return typeof value === 'object' && value !== null && !Array.isArray(value); } +/** + * Strip debug preamble and OpenCode internal log lines from stdout + * so only the actual agent-generated report is returned. + */ +function cleanReport(raw: string): string { + const lines = raw.split('\n'); + let startIdx = 0; + + for (let i = 0; i < lines.length; i++) { + const trimmed = lines[i].trim(); + // Skip [OpenCode] prefixed debug lines + if (trimmed.startsWith('[OpenCode]')) { + startIdx = i + 1; + continue; + } + // Skip MCP server listing box-drawing characters + if ( + /^[┌│└●◆◇]/.test(trimmed) || + trimmed.includes('MCP Servers') || + /^\d+ server\(s\)/.test(trimmed) + ) { + startIdx = i + 1; + continue; + } + // Skip leading blank lines after debug output + if (startIdx > 0 && trimmed === '') { + startIdx = i + 1; + continue; + } + break; + } + + let result = lines.slice(startIdx).join('\n').trim(); + + // Strip any LLM preamble before the first markdown heading (e.g. "## ...") + const headingMatch = result.match(/^(.*?\n)*(#{1,3} )/s); + if (headingMatch && headingMatch.index !== undefined && headingMatch.index > 0) { + const beforeHeading = result.slice(0, headingMatch.index).trim(); + // Only strip if the preamble is short (< 500 chars) — avoids removing real content + if (beforeHeading.length > 0 && beforeHeading.length < 500) { + result = result.slice(result.indexOf(headingMatch[2], headingMatch.index)); + } + } + + return result; +} + function buildProviderEnv(model?: { provider: string; apiKey?: string }): Record { if (!model?.apiKey) { return {}; diff --git a/worker/src/components/github/__tests__/post-pr-comment.test.ts b/worker/src/components/github/__tests__/post-pr-comment.test.ts new file mode 100644 index 000000000..e3d798ccb --- /dev/null +++ b/worker/src/components/github/__tests__/post-pr-comment.test.ts @@ -0,0 +1,226 @@ +import { describe, it, expect, beforeAll, beforeEach, vi, afterEach } from 'bun:test'; +import { createExecutionContext } from '@shipsec/component-sdk'; +import { componentRegistry } from '../../index'; +import type { PostPrCommentInput, PostPrCommentOutput } from '../post-pr-comment'; + +// Mock fetchInstallationToken to return a token without hitting real APIs +vi.mock('../github-auth', () => ({ + sanitizeForLogging: (str: string) => str, + fetchInstallationToken: vi.fn().mockResolvedValue('mock-github-token'), +})); + +// Store the original fetch +const originalFetch = globalThis.fetch; + +describe('github.pr.comment component', () => { + beforeAll(async () => { + await import('../../index'); + }); + + let mockFetch: ReturnType; + + beforeEach(() => { + mockFetch = vi.fn(); + globalThis.fetch = mockFetch; + }); + + afterEach(() => { + globalThis.fetch = originalFetch; + }); + + function getComponent() { + const component = componentRegistry.get( + 'github.pr.comment', + ); + if (!component) throw new Error('Component not registered'); + return component; + } + + function createContext() { + return createExecutionContext({ + runId: 'test-run', + componentRef: 'post-pr-comment-test', + }); + } + + const baseInputs = { + installationId: 12345, + owner: 'myorg', + repo: 'myrepo', + prNumber: 42, + body: '## Scan Results\nNo findings detected.', + }; + + it('should POST a new comment when no existingCommentId is provided', async () => { + const component = getComponent(); + const context = createContext(); + + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => ({ + id: 99001, + html_url: 'https://github.com/myorg/myrepo/pull/42#issuecomment-99001', + }), + }); + + const result = await component.execute({ inputs: baseInputs }, context); + + expect(result.commentId).toBe(99001); + expect(result.commentUrl).toBe('https://github.com/myorg/myrepo/pull/42#issuecomment-99001'); + + // Verify POST was called (not PATCH) + expect(mockFetch).toHaveBeenCalledTimes(1); + const [url, options] = mockFetch.mock.calls[0]; + expect(url).toBe('https://api.github.com/repos/myorg/myrepo/issues/42/comments'); + expect(options.method).toBe('POST'); + expect(JSON.parse(options.body)).toEqual({ body: baseInputs.body }); + }); + + it('should PATCH an existing comment when existingCommentId is provided', async () => { + const component = getComponent(); + const context = createContext(); + + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => ({ + id: 88001, + html_url: 'https://github.com/myorg/myrepo/pull/42#issuecomment-88001', + }), + }); + + const result = await component.execute( + { inputs: { ...baseInputs, existingCommentId: 88001 } }, + context, + ); + + expect(result.commentId).toBe(88001); + + // Verify PATCH was called + expect(mockFetch).toHaveBeenCalledTimes(1); + const [url, options] = mockFetch.mock.calls[0]; + expect(url).toBe('https://api.github.com/repos/myorg/myrepo/issues/comments/88001'); + expect(options.method).toBe('PATCH'); + }); + + it('should fall back to POST when PATCH returns 404', async () => { + const component = getComponent(); + const context = createContext(); + + // PATCH returns 404 + mockFetch.mockResolvedValueOnce({ + ok: false, + status: 404, + text: async () => 'Not Found', + }); + + // POST succeeds + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => ({ + id: 99002, + html_url: 'https://github.com/myorg/myrepo/pull/42#issuecomment-99002', + }), + }); + + const result = await component.execute( + { inputs: { ...baseInputs, existingCommentId: 77001 } }, + context, + ); + + expect(result.commentId).toBe(99002); + + // Verify PATCH was attempted first, then POST + expect(mockFetch).toHaveBeenCalledTimes(2); + const [patchUrl, patchOptions] = mockFetch.mock.calls[0]; + expect(patchUrl).toContain('/issues/comments/77001'); + expect(patchOptions.method).toBe('PATCH'); + + const [postUrl, postOptions] = mockFetch.mock.calls[1]; + expect(postUrl).toBe('https://api.github.com/repos/myorg/myrepo/issues/42/comments'); + expect(postOptions.method).toBe('POST'); + }); + + it('should fall back to POST when PATCH returns 410', async () => { + const component = getComponent(); + const context = createContext(); + + // PATCH returns 410 + mockFetch.mockResolvedValueOnce({ + ok: false, + status: 410, + text: async () => 'Gone', + }); + + // POST succeeds + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => ({ + id: 99003, + html_url: 'https://github.com/myorg/myrepo/pull/42#issuecomment-99003', + }), + }); + + const result = await component.execute( + { inputs: { ...baseInputs, existingCommentId: 77002 } }, + context, + ); + + expect(result.commentId).toBe(99003); + expect(mockFetch).toHaveBeenCalledTimes(2); + expect(mockFetch.mock.calls[0][1].method).toBe('PATCH'); + expect(mockFetch.mock.calls[1][1].method).toBe('POST'); + }); + + it('should fall back to POST when PATCH fails with other error', async () => { + const component = getComponent(); + const context = createContext(); + + // PATCH returns 500 + mockFetch.mockResolvedValueOnce({ + ok: false, + status: 500, + text: async () => 'Internal Server Error', + }); + + // POST succeeds + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => ({ + id: 99004, + html_url: 'https://github.com/myorg/myrepo/pull/42#issuecomment-99004', + }), + }); + + const result = await component.execute( + { inputs: { ...baseInputs, existingCommentId: 77003 } }, + context, + ); + + expect(result.commentId).toBe(99004); + expect(mockFetch).toHaveBeenCalledTimes(2); + }); + + it('should throw when POST fails', async () => { + const component = getComponent(); + const context = createContext(); + + mockFetch.mockResolvedValueOnce({ + ok: false, + status: 403, + text: async () => 'Forbidden', + }); + + await expect(component.execute({ inputs: baseInputs }, context)).rejects.toThrow( + 'Failed to create PR comment', + ); + }); + + it('should throw when installationId is missing', async () => { + const component = getComponent(); + const context = createContext(); + + await expect( + component.execute({ inputs: { ...baseInputs, installationId: undefined } }, context), + ).rejects.toThrow('Installation ID is required'); + }); +}); diff --git a/worker/src/components/github/clone-repo.ts b/worker/src/components/github/clone-repo.ts new file mode 100644 index 000000000..affa92d64 --- /dev/null +++ b/worker/src/components/github/clone-repo.ts @@ -0,0 +1,480 @@ +import { z } from 'zod'; +import { + componentRegistry, + ComponentRetryPolicy, + runComponentWithRunner, + type DockerRunnerConfig, + ContainerError, + defineComponent, + inputs, + outputs, + parameters, + port, + param, +} from '@shipsec/component-sdk'; +import { IsolatedContainerVolume } from '../../utils/isolated-volume'; +import { sanitizeForLogging, fetchInstallationToken } from './github-auth'; + +/** + * Clone Repository Worker Component + * + * Clones a GitHub repository to an isolated Docker volume for security scanning. + * Supports both public repositories and private repositories via GitHub App installation tokens. + */ + +const inputSchema = inputs({ + installationId: port(z.number().optional(), { + label: 'Installation ID', + description: 'GitHub App installation ID for private repos. Leave empty for public repos.', + connectionType: { kind: 'primitive', name: 'number' }, + }), + repository: port(z.string().optional(), { + label: 'Repository', + description: 'Repository in owner/repo format. Overrides the parameter if connected.', + connectionType: { kind: 'primitive', name: 'text' }, + }), + branch: port(z.string().optional(), { + label: 'Branch', + description: 'Branch to clone. Overrides the parameter if connected.', + connectionType: { kind: 'primitive', name: 'text' }, + }), + ref: port(z.string().optional(), { + label: 'Ref', + description: 'Specific commit SHA. Overrides the parameter if connected.', + connectionType: { kind: 'primitive', name: 'text' }, + }), +}); + +const parameterSchema = parameters({ + repository: param( + z.string().trim().optional().describe('Repository to clone in owner/repo format'), + { + label: 'Repository', + editor: 'text', + description: 'Full repository name (e.g., owner/repo). Can also be provided via input port.', + helpText: 'Enter the repository in owner/repo format', + }, + ), + branch: param(z.string().optional().describe('Branch to clone'), { + label: 'Branch', + editor: 'text', + description: 'Branch name to clone (defaults to default branch)', + helpText: 'Leave empty to clone the default branch', + }), + ref: param(z.string().optional().describe('Specific commit SHA or tag'), { + label: 'Ref', + editor: 'text', + description: 'Specific commit SHA or tag to checkout', + helpText: 'Use this for PR scans to checkout the exact commit', + }), + depth: param( + z + .number() + .int() + .min(0) + .max(1000) + .default(1) + .describe('Clone depth (1 for shallow, 0 for full history)'), + { + label: 'Depth', + editor: 'number', + min: 0, + max: 1000, + description: 'Shallow clone depth (1 for latest commit only, 0 for full)', + }, + ), + submodules: param(z.boolean().default(false).describe('Clone submodules recursively'), { + label: 'Include Submodules', + editor: 'boolean', + description: 'Whether to recursively clone submodules', + }), +}); + +const outputSchema = outputs({ + volumePath: port(z.string(), { + label: 'Volume Path', + description: 'Path to the cloned repository in the isolated volume', + connectionType: { kind: 'primitive', name: 'text' }, + }), + repository: port(z.string(), { + label: 'Repository', + description: 'Full repository name that was cloned', + connectionType: { kind: 'primitive', name: 'text' }, + }), + branch: port(z.string(), { + label: 'Branch', + description: 'Branch that was cloned', + connectionType: { kind: 'primitive', name: 'text' }, + }), + commitSha: port(z.string(), { + label: 'Commit SHA', + description: 'The commit SHA that was checked out', + connectionType: { kind: 'primitive', name: 'text' }, + }), + cloneUrl: port(z.string(), { + label: 'Clone URL', + description: 'The public URL of the repository (without token)', + connectionType: { kind: 'primitive', name: 'text' }, + }), + installationId: port(z.number().optional(), { + label: 'Installation ID', + description: 'GitHub App installation ID used for authentication (if resolved)', + connectionType: { kind: 'primitive', name: 'number' }, + }), + volumeName: port(z.string(), { + label: 'Volume Name', + description: + 'Docker volume name containing the cloned repository. Connect this to downstream scanners and to volume-cleanup. The repository is stored at /repo inside the volume.', + connectionType: { kind: 'primitive', name: 'text' }, + }), +}); + +type Output = z.infer; + +// Runner output schema for parsing git command results +const runnerOutputSchema = z.object({ + stdout: z.string().optional().default(''), + stderr: z.string().optional().default(''), + exitCode: z.number().optional().default(0), +}); + +/** + * Validate a git ref (branch name, tag, or commit SHA) to prevent shell injection. + * Allows only characters valid in git refs: alphanumeric, forward slash, dash, + * underscore, dot, and @. Rejects shell metacharacters and traversal patterns. + */ +function validateGitRef(value: string, fieldName: string): void { + if (!/^[a-zA-Z0-9][a-zA-Z0-9/_.@-]*$/.test(value)) { + throw new ContainerError(`Invalid ${fieldName}: contains disallowed characters`, { + details: { reason: 'invalid_git_ref', [fieldName]: value.slice(0, 100) }, + }); + } + if (value.includes('..') || value.includes('@{')) { + throw new ContainerError(`Invalid ${fieldName}: contains disallowed sequence`, { + details: { reason: 'invalid_git_ref', [fieldName]: value.slice(0, 100) }, + }); + } +} + +// Retry policy - git clones can fail due to network issues +const cloneRetryPolicy: ComponentRetryPolicy = { + maxAttempts: 3, + initialIntervalSeconds: 5, + maximumIntervalSeconds: 30, + backoffCoefficient: 2, + nonRetryableErrorTypes: ['ValidationError', 'ConfigurationError'], +}; + +const definition = defineComponent({ + id: 'github.repo.clone', + label: 'Clone GitHub Repository', + category: 'security', + retryPolicy: cloneRetryPolicy, + runner: { + kind: 'docker', + image: 'alpine/git:latest', + entrypoint: 'sh', + network: 'bridge', // Need network for git clone + timeoutSeconds: 300, // 5 minutes for large repos + command: ['-c'], + env: { + HOME: '/tmp', + GIT_TERMINAL_PROMPT: '0', // Disable interactive prompts + }, + }, + inputs: inputSchema, + outputs: outputSchema, + parameters: parameterSchema, + docs: 'Clone a GitHub repository to an isolated Docker volume for security scanning. Supports both public and private repos via GitHub App installation.', + ui: { + slug: 'github-clone-repo', + version: '1.0.0', + type: 'input', + category: 'security', + description: 'Clone a GitHub repository to an isolated Docker volume for security scanning.', + documentation: + 'Clones a repository with configurable depth, branch, and submodule options. For private repos, provide the GitHub App installation ID to authenticate. The cloned code is stored at /repo inside the Docker volume. Downstream components should mount the volume and access the repository at {mountTarget}/repo.', + icon: 'GitBranch', + author: { + name: 'ShipSecAI', + type: 'shipsecai', + }, + isLatest: true, + deprecated: false, + example: 'Clone owner/repo to scan with TruffleHog, OpenGrep, or other scanners.', + examples: [ + 'Clone a repository before running TruffleHog for secret scanning', + 'Clone a PR branch for SAST analysis with OpenGrep', + 'Clone with depth=0 for full git history scanning', + ], + }, + async execute({ inputs, params }, context) { + const parsedInputs = inputSchema.parse(inputs); + const parsedParams = parameterSchema.parse(params); + + // Input ports override parameters (allows wiring from triggers or other components) + const repository = parsedInputs.repository || parsedParams.repository; + const branch = parsedInputs.branch || parsedParams.branch; + const ref = parsedInputs.ref || parsedParams.ref; + const { depth, submodules } = parsedParams; + const { installationId } = parsedInputs; + + if (!repository) { + throw new ContainerError('Repository is required: provide it via input port or parameter'); + } + + // Validate branch/ref to prevent shell injection (defense-in-depth: + // values are also passed via env vars, but reject clearly invalid refs early) + if (branch) { + validateGitRef(branch, 'branch'); + } + if (ref) { + validateGitRef(ref, 'ref'); + } + + context.logger.info(`[GitHub Clone] Starting clone of ${repository}`); + + context.emitProgress({ + message: `Preparing to clone ${repository}...`, + level: 'info', + data: { repository, branch: branch || 'default', stage: 'init' }, + }); + + // Get tenant ID for isolated volume + const tenantId = (context as any).tenantId ?? 'default-tenant'; + let volume: IsolatedContainerVolume | null = null; + + try { + // Create isolated volume for the clone + context.emitProgress({ + message: 'Creating isolated workspace...', + level: 'info', + data: { repository, stage: 'volume' }, + }); + + volume = new IsolatedContainerVolume(tenantId, context.runId); + await volume.initialize({}); + + context.logger.info(`[GitHub Clone] Created isolated volume: ${volume.getVolumeName()}`); + + // Determine clone URL - for private repos, we need the token + const publicCloneUrl = `https://github.com/${repository}.git`; + let gitCloneUrl = publicCloneUrl; + let isAuthenticated = false; + const resolvedInstallationId = installationId; + + context.emitProgress({ + message: 'Authenticating with GitHub...', + level: 'info', + data: { repository, stage: 'auth' }, + }); + + let token: string | null = null; + + if (installationId) { + context.logger.info(`[GitHub Clone] Fetching installation token for ID: ${installationId}`); + token = await fetchInstallationToken(installationId, context, '[GitHub Clone]'); + } else { + context.logger.info(`[GitHub Clone] No installationId provided, attempting public clone`); + } + + if (token) { + gitCloneUrl = `https://x-access-token:${token}@github.com/${repository}.git`; + isAuthenticated = true; + context.logger.info('[GitHub Clone] Successfully authenticated with GitHub App'); + + context.emitProgress({ + message: 'Authenticated with GitHub App', + level: 'info', + data: { repository, authenticated: true, stage: 'auth' }, + }); + } else { + context.logger.warn( + '[GitHub Clone] Could not obtain installation token, attempting clone without authentication', + ); + + context.emitProgress({ + message: 'Warning: Proceeding without authentication (public repos only)', + level: 'warn', + data: { repository, authenticated: false, stage: 'auth' }, + }); + } + + // Emit progress for clone start + context.emitProgress({ + message: `Cloning ${repository}${branch ? ` (branch: ${branch})` : ''}...`, + level: 'info', + data: { repository, branch: branch || 'default', stage: 'clone' }, + }); + + // Build the git clone command. + // SECURITY: Branch, ref, and URL are passed via environment variables + // (double-quoted in the shell) to prevent shell injection. + const cloneArgs: string[] = ['git', 'clone']; + + if (depth > 0) { + cloneArgs.push('--depth', String(depth)); + } + + if (branch) { + cloneArgs.push('--branch', '"$GIT_BRANCH"'); + } + + if (submodules) { + cloneArgs.push('--recurse-submodules'); + } + + // URL is passed via environment variable to avoid logging tokens + cloneArgs.push('"$CLONE_URL"', '/workspace/repo'); + + // Add post-clone commands to get commit info + const postCloneCommands = [ + // If ref is specified, checkout that ref (value via env var) + ref ? 'cd /workspace/repo && git checkout "$GIT_REF"' : '', + // Get the commit SHA + 'cd /workspace/repo && git rev-parse HEAD > /workspace/commit_sha.txt', + // Get the branch name + 'cd /workspace/repo && git rev-parse --abbrev-ref HEAD > /workspace/branch.txt', + ] + .filter(Boolean) + .join(' && '); + + const fullCommand = `${cloneArgs.join(' ')} && ${postCloneCommands}`; + + // Log command without URL (URL would expose token) + context.logger.info( + `[GitHub Clone] Executing: git clone ${depth > 0 ? `--depth ${depth} ` : ''}${branch ? `--branch ${branch} ` : ''}${submodules ? '--recurse-submodules ' : ''} /workspace/repo`, + ); + + // Configure runner + const baseRunner = definition.runner; + if (baseRunner.kind !== 'docker') { + throw new ContainerError('Clone runner must be docker', { + details: { + reason: 'runner_type_mismatch', + expected: 'docker', + actual: baseRunner.kind, + }, + }); + } + + const runnerConfig: DockerRunnerConfig = { + ...baseRunner, + command: ['-c', fullCommand], + env: { + ...baseRunner.env, + // CRITICAL: Pass untrusted values via env vars to prevent shell injection + // and to prevent tokens from appearing in logs. + CLONE_URL: gitCloneUrl, + ...(branch ? { GIT_BRANCH: branch } : {}), + ...(ref ? { GIT_REF: ref } : {}), + }, + volumes: [volume.getVolumeConfig('/workspace', false)], // Read-write for clone + }; + + // Execute the clone + const rawResult = await runComponentWithRunner( + runnerConfig, + async () => ({}) as Output, + { ...parsedInputs, ...parsedParams }, + context, + ); + + // Parse result + const parsedResult = runnerOutputSchema.safeParse(rawResult); + let exitCode = 0; + let stderr = ''; + + if (parsedResult.success) { + exitCode = parsedResult.data.exitCode ?? 0; + stderr = parsedResult.data.stderr ?? ''; + } + + if (exitCode !== 0) { + // CRITICAL: Sanitize error message to remove any tokens before logging/throwing + const sanitizedError = sanitizeForLogging(stderr).slice(0, 500); + + context.emitProgress({ + message: `Clone failed: ${sanitizedError}`, + level: 'error', + data: { repository, exitCode, stage: 'error' }, + }); + + throw new ContainerError(`Git clone failed: ${sanitizedError}`, { + details: { exitCode, repository }, + }); + } + + // Emit progress for post-clone processing + context.emitProgress({ + message: 'Clone complete, extracting commit information...', + level: 'info', + data: { repository, stage: 'post-clone' }, + }); + + // Read commit info from the volume + const files = await volume.readFiles(['commit_sha.txt', 'branch.txt']); + + const commitSha = files['commit_sha.txt']?.trim() || 'unknown'; + const actualBranch = files['branch.txt']?.trim() || branch || 'HEAD'; + + context.logger.info( + `[GitHub Clone] Successfully cloned ${repository} at ${commitSha.substring(0, 7)}`, + ); + + context.emitProgress({ + message: `✓ Clone complete: ${repository} @ ${commitSha.substring(0, 7)}`, + level: 'info', + data: { + repository, + branch: actualBranch, + commitSha, + authenticated: isAuthenticated, + stage: 'complete', + }, + }); + + // Note: We don't cleanup the volume here because downstream components need it + // The workflow engine handles cleanup after the workflow completes + + const output: Output = { + volumePath: '/workspace/repo', + repository, + branch: actualBranch, + commitSha, + cloneUrl: publicCloneUrl, // CRITICAL: Return public URL without token + installationId: resolvedInstallationId, + volumeName: volume!.getVolumeName()!, + }; + + return outputSchema.parse(output); + } catch (error) { + // Clean up volume on error + if (volume) { + await volume.cleanup().catch((cleanupErr) => { + context.logger.warn( + `[GitHub Clone] Failed to cleanup volume: ${cleanupErr instanceof Error ? cleanupErr.message : String(cleanupErr)}`, + ); + }); + } + + if (error instanceof ContainerError) { + throw error; + } + + // CRITICAL: Sanitize any error messages before throwing + const errorMessage = error instanceof Error ? error.message : String(error); + const sanitizedMessage = sanitizeForLogging(errorMessage); + + throw new ContainerError(`Clone failed: ${sanitizedMessage}`, { + cause: error instanceof Error ? error : undefined, + details: { repository }, + }); + } + }, +}); + +componentRegistry.register(definition); + +export type CloneRepoInput = typeof inputSchema; +export type CloneRepoOutput = typeof outputSchema; diff --git a/worker/src/components/github/create-check-run.ts b/worker/src/components/github/create-check-run.ts new file mode 100644 index 000000000..5fefbd33c --- /dev/null +++ b/worker/src/components/github/create-check-run.ts @@ -0,0 +1,209 @@ +import { z } from 'zod'; +import { componentRegistry, defineComponent, inputs, outputs, port } from '@shipsec/component-sdk'; +import { sanitizeForLogging, fetchInstallationToken } from './github-auth'; + +const LOG_PREFIX = '[GitHub Check Run]'; + +/** + * Create Check Run Worker Component + * + * Creates a GitHub Check Run on a commit. Used for showing scan status and results + * directly in the GitHub PR/commit UI. + */ + +// Check run output schema for the optional output field +const checkRunOutputSchema = z + .object({ + title: z.string().describe('Title of the check run output'), + summary: z.string().describe('Summary of the check run (supports markdown)'), + text: z.string().optional().describe('Detailed text (supports markdown)'), + }) + .optional(); + +const inputSchema = inputs({ + installationId: port(z.number().optional(), { + label: 'Installation ID', + description: 'GitHub App installation ID. If not provided, auto-resolved from owner/repo.', + connectionType: { kind: 'primitive', name: 'number' }, + }), + owner: port(z.string(), { + label: 'Owner', + description: 'Repository owner (org or user)', + connectionType: { kind: 'primitive', name: 'text' }, + }), + repo: port(z.string(), { + label: 'Repository', + description: 'Repository name', + connectionType: { kind: 'primitive', name: 'text' }, + }), + headSha: port(z.string(), { + label: 'Head SHA', + description: 'The SHA of the commit to create the check run on', + connectionType: { kind: 'primitive', name: 'text' }, + }), + name: port(z.string(), { + label: 'Check Name', + description: 'The name of the check (e.g., "Security Scan")', + connectionType: { kind: 'primitive', name: 'text' }, + }), + status: port(z.enum(['queued', 'in_progress', 'completed']), { + label: 'Status', + description: 'The current status of the check run', + connectionType: { kind: 'primitive', name: 'text' }, + }), + conclusion: port(z.enum(['success', 'failure', 'neutral', 'cancelled', 'skipped']).optional(), { + label: 'Conclusion', + description: 'The final conclusion of the check run (required when status is completed)', + connectionType: { kind: 'primitive', name: 'text' }, + }), + output: port(checkRunOutputSchema, { + label: 'Output', + description: 'Optional structured output with title, summary, and text', + connectionType: { kind: 'any' }, + }), +}); + +export type CreateCheckRunInput = typeof inputSchema; + +const outputSchema = outputs({ + checkRunId: port(z.number(), { + label: 'Check Run ID', + description: 'GitHub check run ID', + connectionType: { kind: 'primitive', name: 'number' }, + }), +}); + +export type CreateCheckRunOutput = typeof outputSchema; + +const definition = defineComponent({ + id: 'github.check.create', + label: 'Create Check Run', + category: 'notification', + runner: { kind: 'inline' }, + inputs: inputSchema, + outputs: outputSchema, + docs: 'Create a GitHub Check Run to show scan status and results in the PR/commit UI.', + ui: { + slug: 'github-check-create', + version: '1.0.0', + type: 'output', + category: 'notification', + description: 'Create a GitHub Check Run to display scan status and results directly in GitHub.', + icon: 'CheckCircle', + author: { + name: 'ShipSecAI', + type: 'shipsecai', + }, + isLatest: true, + deprecated: false, + examples: [ + 'Create a "Security Scan" check run with queued status', + 'Create a completed check run with findings summary', + ], + }, + async execute({ inputs }, context) { + const { installationId, owner, repo, headSha, name, status, conclusion, output } = inputs; + + context.logger.info( + `${LOG_PREFIX} Creating check run "${name}" for ${owner}/${repo}@${headSha.substring(0, 7)}`, + ); + context.emitProgress({ + message: `Creating check run "${name}"...`, + level: 'info', + data: { owner, repo, headSha: headSha.substring(0, 7), name, stage: 'start' }, + }); + + // Resolve installation token + if (!installationId) { + throw new Error( + `Installation ID is required for ${owner}/${repo}. ` + + 'Ensure the GitHub App installation ID is provided via workflow inputs.', + ); + } + + const token = await fetchInstallationToken(installationId, context, LOG_PREFIX); + + if (!token) { + throw new Error( + `Failed to obtain installation token for ${owner}/${repo}. ` + + 'Ensure the GitHub App is installed and has access to the repository.', + ); + } + + context.emitProgress({ + message: 'Creating check run via GitHub API...', + level: 'info', + data: { owner, repo, name, status, stage: 'creating' }, + }); + + // Build the request body + const requestBody: Record = { + name, + head_sha: headSha, + status, + }; + + // Add conclusion if status is completed + if (status === 'completed') { + if (!conclusion) { + throw new Error( + 'Conclusion is required when status is "completed". ' + + 'Valid values: success, failure, neutral, cancelled, skipped', + ); + } + requestBody.conclusion = conclusion; + } + + // Add optional output + if (output) { + requestBody.output = output; + } + + // Create check run via GitHub API: POST /repos/{owner}/{repo}/check-runs + const apiUrl = `https://api.github.com/repos/${owner}/${repo}/check-runs`; + + const response = await fetch(apiUrl, { + method: 'POST', + headers: { + Authorization: `Bearer ${token}`, + Accept: 'application/vnd.github+json', + 'Content-Type': 'application/json', + 'X-GitHub-Api-Version': '2022-11-28', + }, + body: JSON.stringify(requestBody), + }); + + if (!response.ok) { + const errorText = await response.text(); + const sanitizedError = sanitizeForLogging(errorText); + context.logger.error( + `${LOG_PREFIX} Failed to create check run: ${response.status} ${sanitizedError}`, + ); + + context.emitProgress({ + message: `Failed to create check run: ${response.status}`, + level: 'error', + data: { owner, repo, name, status: response.status, stage: 'error' }, + }); + + throw new Error(`Failed to create check run: ${response.status} ${sanitizedError}`); + } + + const result = (await response.json()) as { id: number }; + const checkRunId = result.id; + + context.logger.info(`${LOG_PREFIX} Created check run ${checkRunId} for ${owner}/${repo}`); + + context.emitProgress({ + message: `Check run created successfully`, + level: 'info', + data: { owner, repo, name, checkRunId, stage: 'complete' }, + }); + + return outputSchema.parse({ + checkRunId, + }); + }, +}); + +componentRegistry.register(definition); diff --git a/worker/src/components/github/github-auth.ts b/worker/src/components/github/github-auth.ts new file mode 100644 index 000000000..58dfe0849 --- /dev/null +++ b/worker/src/components/github/github-auth.ts @@ -0,0 +1,114 @@ +import type { ExecutionContext } from '@shipsec/component-sdk'; + +/** + * Shared GitHub App authentication utilities for worker components. + * + * Provides token resolution via: + * 1. Secrets service (pre-cached tokens) + * 2. HTTP call to backend with installation ID + * 3. HTTP call to backend with repository name (auto-resolution) + */ + +/** API base URL for backend calls */ +export const getApiBaseUrl = (): string => + process.env.STUDIO_API_BASE_URL ?? + process.env.SHIPSEC_API_BASE_URL ?? + process.env.API_BASE_URL ?? + 'http://localhost:3211/api/v1'; + +/** + * Build auth headers for internal worker-to-backend API calls. + * Tries internal service token first, then falls back to Basic Auth. + */ +function getInternalAuthHeaders(): Record { + const internalToken = process.env.INTERNAL_SERVICE_TOKEN; + if (internalToken) { + return { + 'x-internal-token': internalToken, + 'x-organization-id': process.env.ORGANIZATION_ID ?? 'local-dev', + }; + } + + // Fall back to Basic Auth for local development + const username = process.env.STUDIO_API_USERNAME ?? process.env.ADMIN_USERNAME ?? 'admin'; + const password = process.env.STUDIO_API_PASSWORD ?? process.env.ADMIN_PASSWORD ?? 'admin'; + const encoded = + typeof btoa === 'function' + ? btoa(`${username}:${password}`) + : Buffer.from(`${username}:${password}`).toString('base64'); + + return { Authorization: `Basic ${encoded}` }; +} + +/** + * Sanitize any string to remove GitHub access tokens for safe logging. + */ +export function sanitizeForLogging(str: string): string { + return str + .replace(/x-access-token:[^@]+@/g, 'x-access-token:***@') + .replace(/ghp_[a-zA-Z0-9]+/g, 'ghp_***') + .replace(/ghs_[a-zA-Z0-9]+/g, 'ghs_***'); +} + +/** + * Fetch installation token using a known installation ID. + * Tries secrets service first, then falls back to backend HTTP API. + */ +export async function fetchInstallationToken( + installationId: number, + context: ExecutionContext, + logPrefix = '[GitHub]', +): Promise { + // Strategy 1: Try to get from secrets service (pre-cached tokens) + if (context.secrets) { + try { + const tokenSecret = await context.secrets.get(`github-installation-${installationId}`); + if (tokenSecret?.value) { + context.logger.info(`${logPrefix} Retrieved installation token from secrets service`); + return tokenSecret.value; + } + } catch (err) { + context.logger.debug( + `${logPrefix} Secrets service lookup failed: ${err instanceof Error ? err.message : String(err)}`, + ); + } + } + + // Strategy 2: Fetch fresh token via HTTP call to backend API + if (context.http) { + try { + const baseUrl = getApiBaseUrl(); + const tokenUrl = `${baseUrl}/github/installations/${installationId}/token`; + + context.logger.info(`${logPrefix} Fetching installation token from backend API`); + + const response = await context.http.fetch(tokenUrl, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + ...getInternalAuthHeaders(), + }, + }); + + if (response.ok) { + const data = (await response.json()) as { token?: string }; + if (data.token) { + context.logger.info( + `${logPrefix} Successfully retrieved installation token from backend API`, + ); + return data.token; + } + } else { + context.logger.warn( + `${logPrefix} Backend API token request failed with status ${response.status}`, + ); + } + } catch (err) { + context.logger.warn( + `${logPrefix} HTTP token fetch failed: ${err instanceof Error ? err.message : String(err)}`, + ); + } + } + + return null; +} diff --git a/worker/src/components/github/post-pr-comment.ts b/worker/src/components/github/post-pr-comment.ts new file mode 100644 index 000000000..5c34f6edd --- /dev/null +++ b/worker/src/components/github/post-pr-comment.ts @@ -0,0 +1,226 @@ +import { z } from 'zod'; +import { componentRegistry, defineComponent, inputs, outputs, port } from '@shipsec/component-sdk'; +import { sanitizeForLogging, fetchInstallationToken } from './github-auth'; + +const LOG_PREFIX = '[GitHub PR Comment]'; + +/** + * Post PR Comment Worker Component + * + * Posts or updates scan results as a comment on a GitHub pull request. + * Supports upsert: if existingCommentId is provided, attempts PATCH first, + * falling back to POST if the comment no longer exists (404/410). + */ + +const inputSchema = inputs({ + installationId: port(z.number().optional(), { + label: 'Installation ID', + description: 'GitHub App installation ID. If not provided, auto-resolved from owner/repo.', + connectionType: { kind: 'primitive', name: 'number' }, + }), + owner: port(z.string(), { + label: 'Owner', + description: 'Repository owner (org or user)', + connectionType: { kind: 'primitive', name: 'text' }, + }), + repo: port(z.string(), { + label: 'Repository', + description: 'Repository name', + connectionType: { kind: 'primitive', name: 'text' }, + }), + prNumber: port(z.number(), { + label: 'PR Number', + description: 'Pull request number to comment on', + connectionType: { kind: 'primitive', name: 'number' }, + }), + body: port(z.string(), { + label: 'Comment Body', + description: 'Markdown content for the PR comment', + connectionType: { kind: 'primitive', name: 'text' }, + }), + existingCommentId: port(z.number().optional(), { + label: 'Existing Comment ID', + description: 'If provided, updates an existing comment instead of creating a new one.', + connectionType: { kind: 'primitive', name: 'number' }, + }), +}); + +export type PostPrCommentInput = typeof inputSchema; + +const outputSchema = outputs({ + commentId: port(z.number(), { + label: 'Comment ID', + description: 'GitHub comment ID', + connectionType: { kind: 'primitive', name: 'number' }, + }), + commentUrl: port(z.string(), { + label: 'Comment URL', + description: 'Direct URL to the comment', + connectionType: { kind: 'primitive', name: 'text' }, + }), +}); + +export type PostPrCommentOutput = typeof outputSchema; + +const GITHUB_API_HEADERS = { + Accept: 'application/vnd.github+json', + 'Content-Type': 'application/json', + 'X-GitHub-Api-Version': '2022-11-28', +}; + +const definition = defineComponent({ + id: 'github.pr.comment', + label: 'Post PR Comment', + category: 'notification', + runner: { kind: 'inline' }, + inputs: inputSchema, + outputs: outputSchema, + docs: 'Post scan results as a comment on a GitHub pull request.', + ui: { + slug: 'github-pr-comment', + version: '1.0.0', + type: 'output', + category: 'notification', + description: 'Post content as a comment on a GitHub pull request.', + icon: 'MessageSquare', + author: { + name: 'ShipSecAI', + type: 'shipsecai', + }, + isLatest: true, + deprecated: false, + examples: [ + 'Post TruffleHog secret scan results to PR', + 'Post OpenGrep SAST findings summary to PR', + ], + }, + async execute({ inputs }, context) { + const { installationId, owner, repo, prNumber, body, existingCommentId } = inputs; + + context.logger.info(`${LOG_PREFIX} Posting comment to ${owner}/${repo}#${prNumber}`); + context.emitProgress({ + message: `Posting comment to PR #${prNumber}...`, + level: 'info', + data: { owner, repo, prNumber, stage: 'start' }, + }); + + // Resolve installation token + if (!installationId) { + throw new Error( + `Installation ID is required for ${owner}/${repo}. ` + + 'Ensure the GitHub App installation ID is provided via workflow inputs.', + ); + } + + const token = await fetchInstallationToken(installationId, context, LOG_PREFIX); + + if (!token) { + throw new Error( + `Failed to obtain installation token for ${owner}/${repo}. ` + + 'Ensure the GitHub App is installed and has access to the repository.', + ); + } + + const authHeaders = { + Authorization: `Bearer ${token}`, + ...GITHUB_API_HEADERS, + }; + + let commentId: number; + let commentUrl: string; + + // Try PATCH if we have an existing comment ID (upsert mode) + if (existingCommentId) { + context.emitProgress({ + message: 'Updating existing PR comment...', + level: 'info', + data: { owner, repo, prNumber, existingCommentId, stage: 'updating' }, + }); + + const patchUrl = `https://api.github.com/repos/${owner}/${repo}/issues/comments/${existingCommentId}`; + const patchResponse = await fetch(patchUrl, { + method: 'PATCH', + headers: authHeaders, + body: JSON.stringify({ body }), + }); + + if (patchResponse.ok) { + const patchResult = (await patchResponse.json()) as { id: number; html_url: string }; + commentId = patchResult.id; + commentUrl = patchResult.html_url; + + context.logger.info(`${LOG_PREFIX} Updated comment ${commentId} at ${commentUrl}`); + context.emitProgress({ + message: 'Comment updated successfully', + level: 'info', + data: { owner, repo, prNumber, commentId, commentUrl, stage: 'complete' }, + }); + + return outputSchema.parse({ commentId, commentUrl }); + } + + // Fallback to POST if comment was deleted (404/410) + if (patchResponse.status === 404 || patchResponse.status === 410) { + context.logger.warn( + `${LOG_PREFIX} Existing comment ${existingCommentId} not found (${patchResponse.status}), falling back to POST`, + ); + } else { + // Non-retryable PATCH failure — still fall back to POST + const errorText = await patchResponse.text(); + const sanitizedError = sanitizeForLogging(errorText); + context.logger.warn( + `${LOG_PREFIX} PATCH failed (${patchResponse.status}), falling back to POST: ${sanitizedError}`, + ); + } + } + + // POST a new comment + context.emitProgress({ + message: 'Creating PR comment...', + level: 'info', + data: { owner, repo, prNumber, stage: 'creating' }, + }); + + const postUrl = `https://api.github.com/repos/${owner}/${repo}/issues/${prNumber}/comments`; + const postResponse = await fetch(postUrl, { + method: 'POST', + headers: authHeaders, + body: JSON.stringify({ body }), + }); + + if (!postResponse.ok) { + const errorText = await postResponse.text(); + const sanitizedError = sanitizeForLogging(errorText); + context.logger.error( + `${LOG_PREFIX} Failed to create comment: ${postResponse.status} ${sanitizedError}`, + ); + + context.emitProgress({ + message: `Failed to create comment: ${postResponse.status}`, + level: 'error', + data: { owner, repo, prNumber, status: postResponse.status, stage: 'error' }, + }); + + throw new Error(`Failed to create PR comment: ${postResponse.status} ${sanitizedError}`); + } + + const result = (await postResponse.json()) as { id: number; html_url: string }; + commentId = result.id; + commentUrl = result.html_url; + + context.logger.info(`${LOG_PREFIX} Created comment ${commentId} at ${commentUrl}`); + + context.emitProgress({ + message: `Comment posted successfully`, + level: 'info', + data: { owner, repo, prNumber, commentId, commentUrl, stage: 'complete' }, + }); + + return outputSchema.parse({ + commentId, + commentUrl, + }); + }, +}); + +componentRegistry.register(definition); diff --git a/worker/src/components/github/pr-context.ts b/worker/src/components/github/pr-context.ts new file mode 100644 index 000000000..532460a80 --- /dev/null +++ b/worker/src/components/github/pr-context.ts @@ -0,0 +1,328 @@ +import { z } from 'zod'; +import { componentRegistry, defineComponent, inputs, outputs, port } from '@shipsec/component-sdk'; +import { sanitizeForLogging, fetchInstallationToken } from './github-auth'; + +const LOG_PREFIX = '[GitHub PR Context]'; + +/** + * PR Context Worker Component + * + * Fetches metadata and changed files for a GitHub pull request. + * This information is used to: + * - Filter scan findings to only files changed in the PR + * - Build targeted scan configurations + * - Include PR metadata in comments/reports + */ + +// Schema for file change information +const changedFileSchema = z.object({ + filename: z.string().describe('Path to the file'), + status: z + .enum(['added', 'removed', 'modified', 'renamed', 'copied', 'changed', 'unchanged']) + .describe('Type of change'), + additions: z.number().describe('Lines added'), + deletions: z.number().describe('Lines deleted'), + changes: z.number().describe('Total lines changed'), + patch: z.string().optional().describe('Unified diff patch'), +}); + +const inputSchema = inputs({ + installationId: port(z.number(), { + label: 'Installation ID', + description: 'GitHub App installation ID for authentication', + connectionType: { kind: 'primitive', name: 'number' }, + }), + owner: port(z.string(), { + label: 'Owner', + description: 'Repository owner (org or user)', + connectionType: { kind: 'primitive', name: 'text' }, + }), + repo: port(z.string(), { + label: 'Repository', + description: 'Repository name', + connectionType: { kind: 'primitive', name: 'text' }, + }), + pullNumber: port(z.number(), { + label: 'Pull Request Number', + description: 'The pull request number to fetch context for', + connectionType: { kind: 'primitive', name: 'number' }, + }), +}); + +export type PrContextInput = typeof inputSchema; + +const outputSchema = outputs({ + // PR Metadata + title: port(z.string(), { + label: 'PR Title', + description: 'Title of the pull request', + connectionType: { kind: 'primitive', name: 'text' }, + }), + author: port(z.string(), { + label: 'Author', + description: 'GitHub username of the PR author', + connectionType: { kind: 'primitive', name: 'text' }, + }), + baseBranch: port(z.string(), { + label: 'Base Branch', + description: 'Target branch the PR will merge into', + connectionType: { kind: 'primitive', name: 'text' }, + }), + headBranch: port(z.string(), { + label: 'Head Branch', + description: 'Source branch with the changes', + connectionType: { kind: 'primitive', name: 'text' }, + }), + headSha: port(z.string(), { + label: 'Head SHA', + description: 'Commit SHA of the head branch (for cloning/checking)', + connectionType: { kind: 'primitive', name: 'text' }, + }), + baseSha: port(z.string(), { + label: 'Base SHA', + description: 'Commit SHA of the base branch', + connectionType: { kind: 'primitive', name: 'text' }, + }), + + // Changed files + changedFiles: port(z.array(z.string()), { + label: 'Changed Files', + description: 'List of file paths that were changed in this PR', + connectionType: { kind: 'list', element: { kind: 'primitive', name: 'text' } }, + }), + changedFilesDetailed: port(z.array(changedFileSchema), { + label: 'Changed Files (Detailed)', + description: 'Detailed information about each changed file including additions/deletions', + connectionType: { kind: 'any' }, + }), + + // Stats + additions: port(z.number(), { + label: 'Total Additions', + description: 'Total lines added across all files', + connectionType: { kind: 'primitive', name: 'number' }, + }), + deletions: port(z.number(), { + label: 'Total Deletions', + description: 'Total lines deleted across all files', + connectionType: { kind: 'primitive', name: 'number' }, + }), + changedFileCount: port(z.number(), { + label: 'Changed File Count', + description: 'Number of files changed in the PR', + connectionType: { kind: 'primitive', name: 'number' }, + }), + + // Clone info + cloneUrl: port(z.string(), { + label: 'Clone URL', + description: 'HTTPS clone URL for the repository (public, no token)', + connectionType: { kind: 'primitive', name: 'text' }, + }), + + // PR state + state: port(z.string(), { + label: 'PR State', + description: 'Current state of the PR (open, closed, merged)', + connectionType: { kind: 'primitive', name: 'text' }, + }), + isDraft: port(z.boolean(), { + label: 'Is Draft', + description: 'Whether the PR is a draft', + connectionType: { kind: 'primitive', name: 'boolean' }, + }), + labels: port(z.array(z.string()), { + label: 'Labels', + description: 'Labels applied to the PR', + connectionType: { kind: 'list', element: { kind: 'primitive', name: 'text' } }, + }), +}); + +export type PrContextOutput = typeof outputSchema; + +// GitHub API response types +interface GitHubPrResponse { + title: string; + user: { login: string }; + base: { ref: string; sha: string }; + head: { ref: string; sha: string }; + state: string; + draft: boolean; + additions: number; + deletions: number; + changed_files: number; + labels: { name: string }[]; + html_url: string; +} + +interface GitHubFileResponse { + filename: string; + status: 'added' | 'removed' | 'modified' | 'renamed' | 'copied' | 'changed' | 'unchanged'; + additions: number; + deletions: number; + changes: number; + patch?: string; +} + +const definition = defineComponent({ + id: 'github.pr.context', + label: 'Get PR Context', + category: 'security', + runner: { kind: 'inline' }, + inputs: inputSchema, + outputs: outputSchema, + docs: 'Fetch PR metadata and changed files. Use to filter scan results to only changed files.', + ui: { + slug: 'github-pr-context', + version: '1.0.0', + type: 'input', + category: 'security', + description: 'Fetches pull request metadata, changed files, and refs for targeted scanning.', + documentation: + 'Use this component to get PR context before running security scans. The changedFiles output can be used to filter scan results to only show findings in files that were changed in the PR.', + icon: 'GitPullRequest', + author: { + name: 'ShipSecAI', + type: 'shipsecai', + }, + isLatest: true, + deprecated: false, + example: 'Get PR #123 context, then pass changedFiles to filter TruffleHog results', + examples: [ + 'Filter OpenGrep findings to only changed files', + 'Get headSha for cloning the exact PR commit', + 'Check if PR is draft before running expensive scans', + ], + }, + async execute({ inputs }, context) { + const { installationId, owner, repo, pullNumber } = inputSchema.parse(inputs); + + context.logger.info(`${LOG_PREFIX} Fetching context for ${owner}/${repo}#${pullNumber}`); + + context.emitProgress({ + message: `Fetching PR #${pullNumber} context...`, + level: 'info', + data: { owner, repo, pullNumber, stage: 'start' }, + }); + + // Get installation token + const token = await fetchInstallationToken(installationId, context, LOG_PREFIX); + if (!token) { + throw new Error( + `Failed to obtain installation token for ${owner}/${repo}. ` + + 'Ensure the GitHub App is installed and has access.', + ); + } + + const headers = { + Authorization: `Bearer ${token}`, + Accept: 'application/vnd.github+json', + 'X-GitHub-Api-Version': '2022-11-28', + }; + + // Fetch PR metadata + context.emitProgress({ + message: 'Fetching PR metadata...', + level: 'info', + data: { owner, repo, pullNumber, stage: 'metadata' }, + }); + + const prUrl = `https://api.github.com/repos/${owner}/${repo}/pulls/${pullNumber}`; + const prResponse = await fetch(prUrl, { headers }); + + if (!prResponse.ok) { + const errorText = await prResponse.text(); + const sanitizedError = sanitizeForLogging(errorText); + throw new Error(`Failed to fetch PR metadata: ${prResponse.status} ${sanitizedError}`); + } + + const prData = (await prResponse.json()) as GitHubPrResponse; + + // Fetch changed files (paginated - up to 3000 files) + context.emitProgress({ + message: 'Fetching changed files...', + level: 'info', + data: { owner, repo, pullNumber, stage: 'files' }, + }); + + const allFiles: GitHubFileResponse[] = []; + let page = 1; + const perPage = 100; // GitHub max per page + + while (allFiles.length < prData.changed_files && page <= 30) { + const filesUrl = `https://api.github.com/repos/${owner}/${repo}/pulls/${pullNumber}/files?per_page=${perPage}&page=${page}`; + const filesResponse = await fetch(filesUrl, { headers }); + + if (!filesResponse.ok) { + context.logger.warn( + `${LOG_PREFIX} Failed to fetch files page ${page}: ${filesResponse.status}`, + ); + break; + } + + const pageFiles = (await filesResponse.json()) as GitHubFileResponse[]; + if (pageFiles.length === 0) break; + + allFiles.push(...pageFiles); + page++; + } + + context.logger.info( + `${LOG_PREFIX} Fetched ${allFiles.length} changed files for PR #${pullNumber}`, + ); + + // Build outputs + const changedFiles = allFiles.map((f) => f.filename); + const changedFilesDetailed = allFiles.map((f) => ({ + filename: f.filename, + status: f.status, + additions: f.additions, + deletions: f.deletions, + changes: f.changes, + patch: f.patch, + })); + + context.emitProgress({ + message: `✓ PR context fetched: ${allFiles.length} files changed`, + level: 'info', + data: { + owner, + repo, + pullNumber, + changedFileCount: allFiles.length, + additions: prData.additions, + deletions: prData.deletions, + stage: 'complete', + }, + }); + + return outputSchema.parse({ + // PR Metadata + title: prData.title, + author: prData.user.login, + baseBranch: prData.base.ref, + headBranch: prData.head.ref, + headSha: prData.head.sha, + baseSha: prData.base.sha, + + // Changed files + changedFiles, + changedFilesDetailed, + + // Stats + additions: prData.additions, + deletions: prData.deletions, + changedFileCount: allFiles.length, + + // Clone info + cloneUrl: `https://github.com/${owner}/${repo}.git`, + + // PR state + state: prData.state, + isDraft: prData.draft, + labels: prData.labels.map((l) => l.name), + }); + }, +}); + +componentRegistry.register(definition); diff --git a/worker/src/components/github/update-check-run.ts b/worker/src/components/github/update-check-run.ts new file mode 100644 index 000000000..a0a8adef0 --- /dev/null +++ b/worker/src/components/github/update-check-run.ts @@ -0,0 +1,200 @@ +import { z } from 'zod'; +import { componentRegistry, defineComponent, inputs, outputs, port } from '@shipsec/component-sdk'; +import { sanitizeForLogging, fetchInstallationToken } from './github-auth'; + +const LOG_PREFIX = '[GitHub Check Run]'; + +/** + * Update Check Run Worker Component + * + * Updates an existing GitHub Check Run. Used for updating scan status and results + * as scanning progresses or completes. + */ + +// Check run output schema for the optional output field +const checkRunOutputSchema = z + .object({ + title: z.string().describe('Title of the check run output'), + summary: z.string().describe('Summary of the check run (supports markdown)'), + text: z.string().optional().describe('Detailed text (supports markdown)'), + }) + .optional(); + +const inputSchema = inputs({ + installationId: port(z.number().optional(), { + label: 'Installation ID', + description: 'GitHub App installation ID. If not provided, auto-resolved from owner/repo.', + connectionType: { kind: 'primitive', name: 'number' }, + }), + owner: port(z.string(), { + label: 'Owner', + description: 'Repository owner (org or user)', + connectionType: { kind: 'primitive', name: 'text' }, + }), + repo: port(z.string(), { + label: 'Repository', + description: 'Repository name', + connectionType: { kind: 'primitive', name: 'text' }, + }), + checkRunId: port(z.number(), { + label: 'Check Run ID', + description: 'The ID of the check run to update', + connectionType: { kind: 'primitive', name: 'number' }, + }), + status: port(z.enum(['queued', 'in_progress', 'completed']), { + label: 'Status', + description: 'The new status of the check run', + connectionType: { kind: 'primitive', name: 'text' }, + }), + conclusion: port(z.enum(['success', 'failure', 'neutral', 'cancelled', 'skipped']).optional(), { + label: 'Conclusion', + description: 'The final conclusion of the check run (required when status is completed)', + connectionType: { kind: 'primitive', name: 'text' }, + }), + output: port(checkRunOutputSchema, { + label: 'Output', + description: 'Optional structured output with title, summary, and text', + connectionType: { kind: 'any' }, + }), +}); + +export type UpdateCheckRunInput = typeof inputSchema; + +const outputSchema = outputs({ + checkRunId: port(z.number(), { + label: 'Check Run ID', + description: 'GitHub check run ID', + connectionType: { kind: 'primitive', name: 'number' }, + }), +}); + +export type UpdateCheckRunOutput = typeof outputSchema; + +const definition = defineComponent({ + id: 'github.check.update', + label: 'Update Check Run', + category: 'notification', + runner: { kind: 'inline' }, + inputs: inputSchema, + outputs: outputSchema, + docs: 'Update an existing GitHub Check Run to update scan status and results in the PR/commit UI.', + ui: { + slug: 'github-check-update', + version: '1.0.0', + type: 'output', + category: 'notification', + description: + 'Update an existing GitHub Check Run to update scan status and results directly in GitHub.', + icon: 'RefreshCw', + author: { + name: 'ShipSecAI', + type: 'shipsecai', + }, + isLatest: true, + deprecated: false, + examples: [ + 'Update a check run to "in_progress" status', + 'Complete a check run with success conclusion and findings summary', + ], + }, + async execute({ inputs }, context) { + const { installationId, owner, repo, checkRunId, status, conclusion, output } = inputs; + + context.logger.info(`${LOG_PREFIX} Updating check run ${checkRunId} for ${owner}/${repo}`); + context.emitProgress({ + message: `Updating check run ${checkRunId}...`, + level: 'info', + data: { owner, repo, checkRunId, status, stage: 'start' }, + }); + + // Resolve installation token + if (!installationId) { + throw new Error( + `Installation ID is required for ${owner}/${repo}. ` + + 'Ensure the GitHub App installation ID is provided via workflow inputs.', + ); + } + + const token = await fetchInstallationToken(installationId, context, LOG_PREFIX); + + if (!token) { + throw new Error( + `Failed to obtain installation token for ${owner}/${repo}. ` + + 'Ensure the GitHub App is installed and has access to the repository.', + ); + } + + context.emitProgress({ + message: 'Updating check run via GitHub API...', + level: 'info', + data: { owner, repo, checkRunId, status, stage: 'updating' }, + }); + + // Build the request body + const requestBody: Record = { + status, + }; + + // Add conclusion if status is completed + if (status === 'completed') { + if (!conclusion) { + throw new Error( + 'Conclusion is required when status is "completed". ' + + 'Valid values: success, failure, neutral, cancelled, skipped', + ); + } + requestBody.conclusion = conclusion; + } + + // Add optional output + if (output) { + requestBody.output = output; + } + + // Update check run via GitHub API: PATCH /repos/{owner}/{repo}/check-runs/{checkRunId} + const apiUrl = `https://api.github.com/repos/${owner}/${repo}/check-runs/${checkRunId}`; + + const response = await fetch(apiUrl, { + method: 'PATCH', + headers: { + Authorization: `Bearer ${token}`, + Accept: 'application/vnd.github+json', + 'Content-Type': 'application/json', + 'X-GitHub-Api-Version': '2022-11-28', + }, + body: JSON.stringify(requestBody), + }); + + if (!response.ok) { + const errorText = await response.text(); + const sanitizedError = sanitizeForLogging(errorText); + context.logger.error( + `${LOG_PREFIX} Failed to update check run: ${response.status} ${sanitizedError}`, + ); + + context.emitProgress({ + message: `Failed to update check run: ${response.status}`, + level: 'error', + data: { owner, repo, checkRunId, status: response.status, stage: 'error' }, + }); + + throw new Error(`Failed to update check run: ${response.status} ${sanitizedError}`); + } + + const result = (await response.json()) as { id: number }; + + context.logger.info(`${LOG_PREFIX} Updated check run ${checkRunId} for ${owner}/${repo}`); + + context.emitProgress({ + message: `Check run updated successfully`, + level: 'info', + data: { owner, repo, checkRunId, status, stage: 'complete' }, + }); + + return outputSchema.parse({ + checkRunId: result.id, + }); + }, +}); + +componentRegistry.register(definition); diff --git a/worker/src/components/github/volume-cleanup.ts b/worker/src/components/github/volume-cleanup.ts new file mode 100644 index 000000000..48995c842 --- /dev/null +++ b/worker/src/components/github/volume-cleanup.ts @@ -0,0 +1,190 @@ +import { z } from 'zod'; +import { componentRegistry, defineComponent, inputs, outputs, port } from '@shipsec/component-sdk'; +import { exec } from 'child_process'; +import { promisify } from 'util'; + +const execAsync = promisify(exec); + +const LOG_PREFIX = '[Volume Cleanup]'; + +/** + * Volume Cleanup Worker Component + * + * Cleans up Docker volumes created by the github.repo.clone component. + * Should be placed at the end of workflows after all scanners have completed. + * + * This component ensures that cloned repositories don't accumulate on disk, + * preventing storage leaks in long-running deployments. + */ + +const inputSchema = inputs({ + volumeName: port(z.string(), { + label: 'Volume Name', + description: 'Docker volume name to cleanup (from github.repo.clone component)', + connectionType: { kind: 'primitive', name: 'text' }, + }), +}); + +export type VolumeCleanupInput = typeof inputSchema; + +const outputSchema = outputs({ + deleted: port(z.boolean(), { + label: 'Deleted', + description: 'Whether the volume was successfully deleted', + connectionType: { kind: 'primitive', name: 'boolean' }, + }), + volumeName: port(z.string(), { + label: 'Volume Name', + description: 'The volume name that was deleted', + connectionType: { kind: 'primitive', name: 'text' }, + }), + message: port(z.string(), { + label: 'Message', + description: 'Status message describing the cleanup result', + connectionType: { kind: 'primitive', name: 'text' }, + }), +}); + +export type VolumeCleanupOutput = typeof outputSchema; + +const definition = defineComponent({ + id: 'github.repo.volume.cleanup', + label: 'Cleanup Repository Volume', + category: 'security', + runner: { kind: 'inline' }, + inputs: inputSchema, + outputs: outputSchema, + docs: 'Cleans up Docker volumes created by the github.repo.clone component. Place at the end of scan workflows to prevent disk leaks.', + ui: { + slug: 'github-volume-cleanup', + version: '1.0.0', + type: 'output', + category: 'security', + description: 'Remove Docker volumes after security scanning is complete to free disk space.', + documentation: + 'This component should be placed at the end of workflows that use github.repo.clone. It removes the temporary Docker volume containing the cloned repository. Safe to call even if the volume was already deleted.', + icon: 'Trash2', + author: { + name: 'ShipSecAI', + type: 'shipsecai', + }, + isLatest: true, + deprecated: false, + example: + 'Connect volumeName from github.repo.clone to cleanup the cloned repository after scanning.', + examples: [ + 'Clean up after TruffleHog secret scanning', + 'Remove cloned repo after OpenGrep SAST analysis', + 'Free disk space in long-running scan workflows', + ], + }, + async execute({ inputs }, context) { + const { volumeName } = inputSchema.parse(inputs); + + context.logger.info(`${LOG_PREFIX} Starting cleanup of volume: ${volumeName}`); + + context.emitProgress({ + message: `Cleaning up volume ${volumeName}...`, + level: 'info', + data: { volumeName, stage: 'start' }, + }); + + // Validate volume name format to prevent command injection + // Volume names follow pattern: tenant-{tenantId}-run-{runId}-{timestamp} + // Only allow alphanumeric, hyphens, and underscores + const volumeNamePattern = /^[a-zA-Z0-9_-]+$/; + if (!volumeNamePattern.test(volumeName)) { + context.logger.error(`${LOG_PREFIX} Invalid volume name format: ${volumeName}`); + + context.emitProgress({ + message: 'Invalid volume name format', + level: 'error', + data: { volumeName, stage: 'error' }, + }); + + return outputSchema.parse({ + deleted: false, + volumeName, + message: `Invalid volume name format: ${volumeName}`, + }); + } + + try { + // Check if volume exists first + const { stdout: inspectStdout } = await execAsync( + `docker volume inspect ${volumeName} 2>/dev/null || echo "NOT_FOUND"`, + ); + + if (inspectStdout.includes('NOT_FOUND')) { + context.logger.info(`${LOG_PREFIX} Volume ${volumeName} does not exist, skipping cleanup`); + + context.emitProgress({ + message: `Volume ${volumeName} already cleaned up`, + level: 'info', + data: { volumeName, existed: false, stage: 'complete' }, + }); + + return outputSchema.parse({ + deleted: false, + volumeName, + message: `Volume ${volumeName} does not exist (already cleaned up)`, + }); + } + + // Remove the volume + context.logger.info(`${LOG_PREFIX} Removing volume: ${volumeName}`); + + await execAsync(`docker volume rm ${volumeName}`); + + context.logger.info(`${LOG_PREFIX} Successfully removed volume: ${volumeName}`); + + context.emitProgress({ + message: `✓ Volume ${volumeName} cleaned up successfully`, + level: 'info', + data: { volumeName, deleted: true, stage: 'complete' }, + }); + + return outputSchema.parse({ + deleted: true, + volumeName, + message: `Volume ${volumeName} successfully removed`, + }); + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + + // Check if error is because volume is in use + if (errorMessage.includes('volume is in use')) { + context.logger.warn(`${LOG_PREFIX} Volume ${volumeName} is still in use, cannot remove`); + + context.emitProgress({ + message: `Volume ${volumeName} is still in use`, + level: 'warn', + data: { volumeName, inUse: true, stage: 'error' }, + }); + + return outputSchema.parse({ + deleted: false, + volumeName, + message: `Volume ${volumeName} is still in use by a container`, + }); + } + + context.logger.error(`${LOG_PREFIX} Failed to remove volume ${volumeName}: ${errorMessage}`); + + context.emitProgress({ + message: `Failed to cleanup volume: ${errorMessage}`, + level: 'error', + data: { volumeName, error: errorMessage, stage: 'error' }, + }); + + // Don't throw - cleanup failures shouldn't fail the workflow + return outputSchema.parse({ + deleted: false, + volumeName, + message: `Failed to remove volume: ${errorMessage}`, + }); + } + }, +}); + +componentRegistry.register(definition); diff --git a/worker/src/components/index.ts b/worker/src/components/index.ts index 714c92e28..19ee31488 100644 --- a/worker/src/components/index.ts +++ b/worker/src/components/index.ts @@ -56,10 +56,18 @@ import './security/terminal-demo'; import './security/virustotal'; import './security/abuseipdb'; import './security/aws-mcp-group'; +import './security/findings-normalize'; +import './security/findings-markdown'; // GitHub components import './github/connection-provider'; import './github/remove-org-membership'; +import './github/clone-repo'; +import './github/post-pr-comment'; +import './github/create-check-run'; +import './github/update-check-run'; +import './github/volume-cleanup'; +import './github/pr-context'; // IT Automation components import './it-automation/google-workspace-license-unassign'; @@ -74,5 +82,23 @@ import './test/live-event-heartbeat'; import './test/simple-http-mcp'; import './test/analytics-fixture'; +// Scanner components (GitHub integration) +import './scanners/trufflehog'; +import './scanners/opengrep'; +import './scanners/dependency'; +import './scanners/trivy'; + +// Scanner components (GitHub integration) +import './scanners/trufflehog'; +import './scanners/opengrep'; +import './scanners/dependency'; +import './scanners/trivy'; + +// Scanner components (GitHub integration) +import './scanners/trufflehog'; +import './scanners/opengrep'; +import './scanners/dependency'; +import './scanners/trivy'; + // Export registry for external use export { componentRegistry } from '@shipsec/component-sdk'; diff --git a/worker/src/components/scanners/__tests__/trufflehog.test.ts b/worker/src/components/scanners/__tests__/trufflehog.test.ts new file mode 100644 index 000000000..533b9eeb4 --- /dev/null +++ b/worker/src/components/scanners/__tests__/trufflehog.test.ts @@ -0,0 +1,185 @@ +import { beforeAll, afterEach, describe, expect, it, vi } from 'bun:test'; +import * as sdk from '@shipsec/component-sdk'; +import { componentRegistry } from '../../index'; + +describe('scanner.trufflehog component', () => { + beforeAll(async () => { + await import('../../index'); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it('should not pass --results flag when onlyVerified is false', async () => { + const component = componentRegistry.get('scanner.trufflehog'); + if (!component) throw new Error('Component not registered'); + + const context = sdk.createExecutionContext({ + runId: 'test-run', + componentRef: 'trufflehog-scanner-test', + }); + + const executePayload = { + inputs: { + volumeName: 'test-volume', + }, + params: { + onlyVerified: false, + }, + }; + + const spy = vi.spyOn(sdk, 'runComponentWithRunner').mockResolvedValue(''); + + await component.execute(executePayload, context); + + const runnerConfig = spy.mock.calls[0]?.[0] as { command?: string[] }; + const command = runnerConfig.command ?? []; + expect(command.some((arg) => arg.startsWith('--results='))).toBe(false); + }); + + it('should pass --results=verified when onlyVerified is true', async () => { + const component = componentRegistry.get('scanner.trufflehog'); + if (!component) throw new Error('Component not registered'); + + const context = sdk.createExecutionContext({ + runId: 'test-run', + componentRef: 'trufflehog-scanner-test', + }); + + const executePayload = { + inputs: { + volumeName: 'test-volume', + }, + params: { + onlyVerified: true, + }, + }; + + const spy = vi.spyOn(sdk, 'runComponentWithRunner').mockResolvedValue(''); + + await component.execute(executePayload, context); + + const runnerConfig = spy.mock.calls[0]?.[0] as { command?: string[] }; + const command = runnerConfig.command ?? []; + expect(command).toContain('--results=verified'); + }); + + it('should use shell wrapper command and append dynamic args', async () => { + const component = componentRegistry.get('scanner.trufflehog'); + if (!component) throw new Error('Component not registered'); + + const context = sdk.createExecutionContext({ + runId: 'test-run', + componentRef: 'trufflehog-scanner-test', + }); + + const executePayload = { + inputs: { + volumeName: 'test-volume', + }, + params: {}, + }; + + const spy = vi.spyOn(sdk, 'runComponentWithRunner').mockResolvedValue(''); + + await component.execute(executePayload, context); + + const runnerConfig = spy.mock.calls[0]?.[0] as { command?: string[]; entrypoint?: string }; + const command = runnerConfig.command ?? []; + + expect(runnerConfig.entrypoint).toBe('sh'); + expect(command[0]).toBe('-c'); + expect(command[1]).toContain('trufflehog "$@" 2>&1'); + expect(command).toContain('filesystem'); + expect(command).toContain('/scan/repo'); + }); + + it('should parse numeric DetectorType using DetectorName without throwing', async () => { + const component = componentRegistry.get('scanner.trufflehog'); + if (!component) throw new Error('Component not registered'); + + const context = sdk.createExecutionContext({ + runId: 'test-run', + componentRef: 'trufflehog-scanner-test', + }); + + const executePayload = { + inputs: { + volumeName: 'test-volume', + }, + params: {}, + }; + + const rawOutput = [ + JSON.stringify({ + SourceMetadata: { Data: { Filesystem: { file: '/scan/repo/.git/config', line: 7 } } }, + DetectorType: 8, + DetectorName: 'Github', + Verified: false, + Raw: 'ghs_example', + Redacted: '', + }), + ].join('\n'); + + vi.spyOn(sdk, 'runComponentWithRunner').mockResolvedValue(rawOutput); + + const output = (await component.execute(executePayload, context)) as { + findingCount: number; + findings: { + type?: string; + secret_type?: string; + severity?: string; + }[]; + }; + + expect(output.findingCount).toBe(1); + expect(output.findings[0]?.type).toBe('Github'); + expect(output.findings[0]?.secret_type).toBe('Github'); + expect(output.findings[0]?.severity).toBe('medium'); + }); + + it('should ignore non-finding status JSON lines', async () => { + const component = componentRegistry.get('scanner.trufflehog'); + if (!component) throw new Error('Component not registered'); + + const context = sdk.createExecutionContext({ + runId: 'test-run', + componentRef: 'trufflehog-scanner-test', + }); + + const executePayload = { + inputs: { + volumeName: 'test-volume', + }, + params: {}, + }; + + const rawOutput = [ + JSON.stringify({ level: 'info-0', msg: 'running source', trufflehog_version: '3.93.1' }), + JSON.stringify({ + SourceMetadata: { + Data: { Filesystem: { file: '/scan/repo/fake-aws-tokens.txt', line: 2 } }, + }, + DetectorType: 2, + DetectorName: 'AWS', + Verified: false, + Raw: 'AKIAEXAMPLE', + }), + ].join('\n'); + + vi.spyOn(sdk, 'runComponentWithRunner').mockResolvedValue(rawOutput); + + const output = (await component.execute(executePayload, context)) as { + findingCount: number; + findings: { + type?: string; + severity?: string; + }[]; + }; + + expect(output.findingCount).toBe(1); + expect(output.findings[0]?.type).toBe('AWS'); + expect(output.findings[0]?.severity).toBe('high'); + }); +}); diff --git a/worker/src/components/scanners/dependency.ts b/worker/src/components/scanners/dependency.ts new file mode 100644 index 000000000..120465f09 --- /dev/null +++ b/worker/src/components/scanners/dependency.ts @@ -0,0 +1,380 @@ +import { z } from 'zod'; +import { + componentRegistry, + ComponentRetryPolicy, + runComponentWithRunner, + type DockerRunnerConfig, + ContainerError, + defineComponent, + inputs, + outputs, + parameters, + port, + param, +} from '@shipsec/component-sdk'; + +/** + * Dependency Scanner Component (using safedep/vet) + * + * Scans dependencies for known vulnerabilities using safedep/vet. + * Auto-detects package managers from lockfiles. + */ + +// Finding type for standardized output format +const findingSchema = z.object({ + package: z.string().describe('Package name'), + version: z.string().describe('Installed version'), + vulnerability_id: z.string().describe('CVE or vulnerability ID'), + severity: z.enum(['critical', 'high', 'medium', 'low', 'info']).describe('Severity level'), + fixed_version: z.string().nullable().describe('Version that fixes the vulnerability'), + description: z.string().describe('Description of the vulnerability'), +}); + +type Finding = z.infer; + +// Input schema - receives volumeName from clone component +const inputSchema = inputs({ + volumeName: port( + z + .string() + .min(1, 'Volume name is required') + .describe('Docker volume name containing the cloned repository'), + { + label: 'Volume Name', + description: + 'Docker volume name from github.repo.clone. The repository is at /repo inside the volume.', + connectionType: { kind: 'primitive', name: 'text' }, + }, + ), +}); + +// Configuration parameters +const parameterSchema = parameters({ + failOnVuln: param( + z.boolean().default(false).describe('Fail the scan if vulnerabilities are found'), + { + label: 'Fail on Vulnerabilities', + editor: 'boolean', + description: 'Exit with error code if vulnerabilities are detected.', + }, + ), + severityThreshold: param( + z + .enum(['critical', 'high', 'medium', 'low']) + .default('high') + .describe('Minimum severity to report'), + { + label: 'Severity Threshold', + editor: 'select', + options: [ + { label: 'Critical only', value: 'critical' }, + { label: 'High and above', value: 'high' }, + { label: 'Medium and above', value: 'medium' }, + { label: 'All vulnerabilities', value: 'low' }, + ], + description: 'Only report vulnerabilities at or above this severity.', + }, + ), + ecosystems: param( + z + .array(z.enum(['npm', 'pypi', 'cargo', 'go', 'maven', 'nuget', 'rubygems'])) + .optional() + .describe('Ecosystems to scan'), + { + label: 'Ecosystems', + editor: 'multi-select', + options: [ + { label: 'npm (JavaScript/TypeScript)', value: 'npm' }, + { label: 'PyPI (Python)', value: 'pypi' }, + { label: 'Cargo (Rust)', value: 'cargo' }, + { label: 'Go modules', value: 'go' }, + { label: 'Maven (Java)', value: 'maven' }, + { label: 'NuGet (.NET)', value: 'nuget' }, + { label: 'RubyGems (Ruby)', value: 'rubygems' }, + ], + description: 'Limit scan to specific ecosystems. Leave empty to auto-detect.', + }, + ), +}); + +// Output schema - standardized findings format +const outputSchema = outputs({ + findings: port(z.array(findingSchema), { + label: 'Findings', + description: 'Array of vulnerable dependencies in standardized format.', + connectionType: { kind: 'list', element: { kind: 'primitive', name: 'json' } }, + }), + findingCount: port(z.number(), { + label: 'Finding Count', + description: 'Total number of vulnerable dependencies.', + }), + packageCount: port(z.number(), { + label: 'Package Count', + description: 'Total number of packages scanned.', + }), + rawOutput: port(z.string(), { + label: 'Raw Output', + description: 'Raw vet JSON output for debugging.', + }), +}); + +type Output = z.infer; + +// safedep/vet output format +interface VetVulnerability { + id: string; + package: string; + version: string; + severity: string; + title: string; + description: string; + fixed_version?: string; + reference?: string; +} + +interface VetOutput { + packages?: number; + vulnerabilities?: VetVulnerability[]; + summary?: { + total: number; + critical: number; + high: number; + medium: number; + low: number; + }; +} + +// Map vet severity to standardized severity +function mapSeverity(vetSeverity: string): Finding['severity'] { + switch (vetSeverity.toLowerCase()) { + case 'critical': + return 'critical'; + case 'high': + return 'high'; + case 'moderate': + case 'medium': + return 'medium'; + case 'low': + return 'low'; + default: + return 'info'; + } +} + +// Convert vet output to standardized findings +function convertToFindings(vulnerabilities: VetVulnerability[]): Finding[] { + return vulnerabilities.map((vuln) => ({ + package: vuln.package, + version: vuln.version, + vulnerability_id: vuln.id, + severity: mapSeverity(vuln.severity), + fixed_version: vuln.fixed_version || null, + description: vuln.description || vuln.title, + })); +} + +// Parse vet JSON output +function parseVetOutput(rawOutput: string): VetOutput { + if (!rawOutput || rawOutput.trim().length === 0) { + return { packages: 0, vulnerabilities: [] }; + } + + // vet outputs JSON lines, try to parse the last valid JSON object + const lines = rawOutput.split('\n').filter((line) => line.trim().length > 0); + + // Try to find a complete JSON output + for (const line of lines.reverse()) { + try { + const parsed = JSON.parse(line); + if (parsed && (parsed.packages !== undefined || parsed.vulnerabilities !== undefined)) { + return parsed; + } + } catch { + continue; + } + } + + // Try to parse as single JSON blob + try { + return JSON.parse(rawOutput); + } catch { + return { packages: 0, vulnerabilities: [] }; + } +} + +// Retry policy for vet +const dependencyRetryPolicy: ComponentRetryPolicy = { + maxAttempts: 2, + initialIntervalSeconds: 5, + maximumIntervalSeconds: 30, + backoffCoefficient: 2, + nonRetryableErrorTypes: ['ContainerError', 'ValidationError', 'ConfigurationError'], +}; + +const definition = defineComponent({ + id: 'scanner.dependency', + label: 'Dependency Scanner', + category: 'scanners', + retryPolicy: dependencyRetryPolicy, + runner: { + kind: 'docker', + image: 'ghcr.io/safedep/vet:latest', + entrypoint: 'vet', + network: 'bridge', // Needs network for vulnerability database lookups + timeoutSeconds: 300, + command: [], + env: { + HOME: '/tmp', + }, + }, + inputs: inputSchema, + outputs: outputSchema, + parameters: parameterSchema, + docs: 'Scan dependencies for known vulnerabilities using safedep/vet. Auto-detects package managers (npm, pip, cargo, go, maven) from lockfiles.', + ui: { + slug: 'dependency-scanner', + version: '1.0.0', + type: 'scan', + category: 'scanners', + description: + 'Scan project dependencies for known vulnerabilities using the safedep/vet security scanner.', + documentation: + 'vet scans your project dependencies against vulnerability databases to find known CVEs. Supports npm, pip, cargo, go, maven, nuget, and rubygems.', + documentationUrl: 'https://github.com/safedep/vet', + icon: 'Package', + author: { + name: 'ShipSecAI', + type: 'shipsecai', + }, + isLatest: true, + deprecated: false, + example: + 'Clone a repository with github.repo.clone, then connect volumeName to this component for dependency scanning.', + examples: [ + 'Scan npm packages in package-lock.json for known CVEs.', + 'Check Python dependencies in requirements.txt for vulnerabilities.', + 'Audit Cargo.lock for vulnerable Rust crates.', + 'Identify vulnerable Go modules in go.sum.', + ], + }, + async execute({ inputs, params }, context) { + const parsedInputs = inputSchema.parse(inputs); + const parsedParams = parameterSchema.parse(params); + + context.logger.info(`[Dependency Scanner] Starting scan on volume: ${parsedInputs.volumeName}`); + + context.emitProgress({ + message: 'Launching dependency vulnerability scan...', + level: 'info', + data: { volumeName: parsedInputs.volumeName }, + }); + + // Build vet command arguments + const args: string[] = [ + 'scan', + '--json', + '-D', // Scan directory + '/scan', + ]; + + // Add severity threshold + if (parsedParams.severityThreshold) { + args.push('--filter-severity', parsedParams.severityThreshold); + } + + // Add fail on vuln flag + if (parsedParams.failOnVuln) { + args.push('--fail-on-vuln'); + } + + context.logger.info(`[Dependency Scanner] Command: vet ${args.join(' ')}`); + + const baseRunner = definition.runner; + if (baseRunner.kind !== 'docker') { + throw new ContainerError('Dependency scanner runner must be docker', { + details: { reason: 'runner_type_mismatch', expected: 'docker', actual: baseRunner.kind }, + }); + } + + // Configure runner with volume mount + const runnerConfig: DockerRunnerConfig = { + ...baseRunner, + command: args, + volumes: [ + { + source: parsedInputs.volumeName, + target: '/scan', + readOnly: true, + }, + ], + }; + + // Execute vet + let rawResult: unknown; + try { + rawResult = await runComponentWithRunner( + runnerConfig, + async () => ({}) as Output, + { ...parsedInputs, ...parsedParams }, + context, + ); + } catch (error) { + // vet may exit with non-zero if vulnerabilities are found and failOnVuln is set + const errorMessage = error instanceof Error ? error.message : String(error); + context.logger.warn(`[Dependency Scanner] vet exited with error: ${errorMessage}`); + // Continue processing to extract any output + rawResult = ''; + } + + // Parse output + const rawOutput = typeof rawResult === 'string' ? rawResult : ''; + const vetOutput = parseVetOutput(rawOutput); + const findings = convertToFindings(vetOutput.vulnerabilities || []); + + context.logger.info( + `[Dependency Scanner] Found ${findings.length} vulnerable dependencies in ${vetOutput.packages || 0} packages`, + ); + + // Count by severity + const severityCounts = findings.reduce( + (acc, f) => { + acc[f.severity] = (acc[f.severity] || 0) + 1; + return acc; + }, + {} as Record, + ); + + // Emit progress with results summary + if (findings.length > 0) { + const criticalHigh = (severityCounts.critical || 0) + (severityCounts.high || 0); + context.emitProgress({ + message: + criticalHigh > 0 + ? `⚠️ Found ${criticalHigh} critical/high severity vulnerabilities` + : `Found ${findings.length} vulnerable dependencies`, + level: criticalHigh > 0 ? 'warn' : 'info', + data: { findingCount: findings.length, severityCounts }, + }); + } else { + context.emitProgress({ + message: 'No vulnerable dependencies detected', + level: 'info', + }); + } + + const output: Output = { + findings, + findingCount: findings.length, + packageCount: vetOutput.packages || 0, + rawOutput, + }; + + return outputSchema.parse(output); + }, +}); + +componentRegistry.register(definition); + +// Export types for external use +export type DependencyScannerInput = typeof inputSchema; +export type DependencyScannerOutput = typeof outputSchema; diff --git a/worker/src/components/scanners/opengrep.ts b/worker/src/components/scanners/opengrep.ts new file mode 100644 index 000000000..7d49aafb7 --- /dev/null +++ b/worker/src/components/scanners/opengrep.ts @@ -0,0 +1,465 @@ +import { z } from 'zod'; +import { + componentRegistry, + ComponentRetryPolicy, + runComponentWithRunner, + type DockerRunnerConfig, + ContainerError, + defineComponent, + inputs, + outputs, + parameters, + port, + param, +} from '@shipsec/component-sdk'; + +/** + * OpenGrep SAST Scanner Component + * + * Performs static application security testing (SAST) on a cloned repository. + * Designed to work with the GitHub clone-repo component via volumeName. + */ + +// Finding type for standardized output format +const findingSchema = z.object({ + rule_id: z.string().describe('OpenGrep rule ID that triggered the finding'), + severity: z.enum(['critical', 'high', 'medium', 'low', 'info']).describe('Severity level'), + file: z.string().describe('File path where the issue was found'), + line: z.number().describe('Line number in the file'), + message: z.string().describe('Description of the security issue'), + snippet: z.string().describe('Code snippet containing the finding'), +}); + +type Finding = z.infer; + +// Input schema - receives volumeName from clone component +const inputSchema = inputs({ + volumeName: port( + z + .string() + .min(1, 'Volume name is required') + .describe('Docker volume name containing the cloned repository'), + { + label: 'Volume Name', + description: + 'Docker volume name from github.repo.clone. The repository is at /repo inside the volume.', + connectionType: { kind: 'primitive', name: 'text' }, + }, + ), + changedFiles: port( + z + .array(z.string()) + .optional() + .describe('List of changed file paths to scope the scan to (e.g. from PR context).'), + { + label: 'Changed Files', + description: + 'Optional list of file paths changed in the PR. When provided, only these files are scanned instead of the entire repository.', + connectionType: { kind: 'list', element: { kind: 'primitive', name: 'text' } }, + allowAny: true, + reason: 'Accepts any array of file path strings.', + }, + ), +}); + +// Configuration parameters +const parameterSchema = parameters({ + ruleset: param( + z + .enum(['auto', 'p/default', 'p/security-audit', 'p/owasp-top-ten', 'p/cwe-top-25']) + .default('auto') + .describe('OpenGrep ruleset to use'), + { + label: 'Ruleset', + editor: 'select', + options: [ + { label: 'Auto-detect + Security', value: 'auto' }, + { label: 'Default Rules', value: 'p/default' }, + { label: 'Security Audit', value: 'p/security-audit' }, + { label: 'OWASP Top 10', value: 'p/owasp-top-ten' }, + { label: 'CWE Top 25', value: 'p/cwe-top-25' }, + ], + description: 'OpenGrep ruleset to apply. Auto will detect language and apply security rules.', + }, + ), + severityFilter: param( + z + .array(z.enum(['ERROR', 'WARNING', 'INFO'])) + .optional() + .describe('Filter results by severity'), + { + label: 'Severity Filter', + editor: 'multi-select', + options: [ + { label: 'Error (High/Critical)', value: 'ERROR' }, + { label: 'Warning (Medium)', value: 'WARNING' }, + { label: 'Info (Low)', value: 'INFO' }, + ], + description: 'Only show findings at these severity levels.', + }, + ), + excludePaths: param(z.array(z.string()).optional().describe('Paths to exclude from scanning'), { + label: 'Exclude Paths', + editor: 'tags', + placeholder: 'node_modules, vendor, .git', + description: 'Glob patterns for paths to exclude from scanning.', + }), + timeout: param( + z.number().int().min(60).max(1800).default(300).describe('Scan timeout in seconds'), + { + label: 'Timeout (seconds)', + editor: 'number', + min: 60, + max: 1800, + description: 'Maximum time for the scan to complete.', + }, + ), + jobs: param(z.number().int().min(1).max(16).default(4).describe('Number of parallel jobs'), { + label: 'Parallel Jobs', + editor: 'number', + min: 1, + max: 16, + description: 'Number of parallel scanning jobs.', + }), +}); + +// Output schema - standardized findings format +const outputSchema = outputs({ + findings: port(z.array(findingSchema), { + label: 'Findings', + description: 'Array of detected security issues in standardized format.', + connectionType: { kind: 'list', element: { kind: 'primitive', name: 'json' } }, + }), + findingCount: port(z.number(), { + label: 'Finding Count', + description: 'Total number of issues detected.', + }), + rawOutput: port(z.string(), { + label: 'Raw Output', + description: 'Raw OpenGrep JSON output for debugging.', + }), +}); + +type Output = z.infer; + +// OpenGrep raw output format (compatible with Semgrep JSON schema) +interface OpenGrepResult { + check_id: string; + path: string; + start: { + line: number; + col: number; + }; + end: { + line: number; + col: number; + }; + extra: { + message: string; + severity: string; + lines?: string; + metadata?: { + cwe?: string[]; + owasp?: string[]; + category?: string; + }; + }; +} + +interface OpenGrepOutput { + results?: OpenGrepResult[]; + errors?: { + message: string; + level: string; + }[]; +} + +// Map OpenGrep severity to standardized severity +function mapSeverity(ogSeverity: string): Finding['severity'] { + switch (ogSeverity.toUpperCase()) { + case 'ERROR': + return 'high'; + case 'WARNING': + return 'medium'; + case 'INFO': + return 'low'; + default: + return 'info'; + } +} + +// Known workspace prefixes that scanners add to file paths. +const WORKSPACE_PREFIXES = ['/workspace/repo/', '/scan/repo/', '/scan/', '/workspace/', '/repo/']; + +function stripWorkspacePrefix(filePath: string): string { + for (const prefix of WORKSPACE_PREFIXES) { + if (filePath.startsWith(prefix)) { + return filePath.slice(prefix.length); + } + } + if (filePath.startsWith('/')) { + return filePath.slice(1); + } + return filePath; +} + +// Convert OpenGrep output to standardized findings +function convertToFindings(results: OpenGrepResult[]): Finding[] { + return results.map((result) => ({ + rule_id: result.check_id, + severity: mapSeverity(result.extra.severity), + file: stripWorkspacePrefix(result.path), + line: result.start.line, + message: result.extra.message, + snippet: result.extra.lines || '', + })); +} + +// Parse OpenGrep JSON output +function parseOpenGrepOutput(rawOutput: string): OpenGrepOutput { + if (!rawOutput || rawOutput.trim().length === 0) { + return { results: [] }; + } + + try { + return JSON.parse(rawOutput); + } catch { + // Try to find JSON in the output (OpenGrep may output non-JSON messages first) + const jsonMatch = rawOutput.match(/\{[\s\S]*"results"[\s\S]*\}/); + if (jsonMatch) { + try { + return JSON.parse(jsonMatch[0]); + } catch { + return { results: [] }; + } + } + return { results: [] }; + } +} + +// Retry policy for OpenGrep +const opengrepRetryPolicy: ComponentRetryPolicy = { + maxAttempts: 2, + initialIntervalSeconds: 10, + maximumIntervalSeconds: 60, + backoffCoefficient: 2, + nonRetryableErrorTypes: ['ContainerError', 'ValidationError', 'ConfigurationError'], +}; + +const definition = defineComponent({ + id: 'scanner.opengrep', + label: 'OpenGrep SAST Scanner', + category: 'scanners', + retryPolicy: opengrepRetryPolicy, + runner: { + kind: 'docker', + image: 'realgam3/opengrep:latest', + entrypoint: 'opengrep', + network: 'bridge', // May need network for ruleset downloads + timeoutSeconds: 600, + command: [], + env: { + HOME: '/tmp', + SEMGREP_SEND_METRICS: 'off', + }, + }, + inputs: inputSchema, + outputs: outputSchema, + parameters: parameterSchema, + docs: 'Perform static application security testing (SAST) on a cloned repository using OpenGrep. Works with the github.repo.clone component.', + ui: { + slug: 'opengrep-scanner', + version: '1.0.0', + type: 'scan', + category: 'scanners', + description: + 'Static code analysis for security vulnerabilities using OpenGrep with 2000+ community rules.', + documentation: + 'OpenGrep is a fast, open-source static analysis tool (Semgrep community fork) that finds security bugs, code quality issues, and enforces coding standards. Auto-detects languages and applies security rulesets.', + documentationUrl: 'https://github.com/opengrep/opengrep', + icon: 'Shield', + author: { + name: 'ShipSecAI', + type: 'shipsecai', + }, + isLatest: true, + deprecated: false, + example: + 'Clone a repository with github.repo.clone, then connect volumeName to this component for SAST scanning.', + examples: [ + 'Scan for SQL injection vulnerabilities in a web application.', + 'Check for insecure deserialization in Python code.', + 'Detect hardcoded credentials and secrets.', + 'Enforce secure coding standards across the codebase.', + ], + }, + async execute({ inputs, params }, context) { + const parsedInputs = inputSchema.parse(inputs); + const parsedParams = parameterSchema.parse(params); + + context.logger.info( + `[OpenGrep Scanner] Starting SAST scan on volume: ${parsedInputs.volumeName}`, + ); + + context.emitProgress({ + message: 'Launching OpenGrep SAST scan...', + level: 'info', + data: { volumeName: parsedInputs.volumeName, ruleset: parsedParams.ruleset }, + }); + + // Build OpenGrep command arguments + const args: string[] = [ + 'scan', + '--json', + '--no-git-ignore', // Don't use git ignore (volume may not have .git) + '--metrics=off', + ]; + + // Add ruleset configuration + // Note: --config auto requires metrics enabled. + // To avoid sending telemetry, we use explicit rulesets that provide equivalent coverage. + if (parsedParams.ruleset === 'auto') { + args.push('--config', 'p/default'); + args.push('--config', 'p/security-audit'); + } else { + args.push('--config', parsedParams.ruleset); + } + + // Add severity filter + if (parsedParams.severityFilter && parsedParams.severityFilter.length > 0) { + for (const sev of parsedParams.severityFilter) { + args.push('--severity', sev); + } + } + + // Add exclude paths + if (parsedParams.excludePaths && parsedParams.excludePaths.length > 0) { + for (const excludePath of parsedParams.excludePaths) { + args.push('--exclude', excludePath); + } + } + + // Add timeout and jobs + args.push('--timeout', parsedParams.timeout.toString()); + args.push('--jobs', parsedParams.jobs.toString()); + + // Scope scan to changed files only (if provided) + if (parsedInputs.changedFiles && parsedInputs.changedFiles.length > 0) { + for (const file of parsedInputs.changedFiles) { + args.push('--include', file); + } + context.logger.info( + `[OpenGrep Scanner] Scoping to ${parsedInputs.changedFiles.length} changed files`, + ); + } + + // Scan target: volume is mounted at /workspace, repo is at /repo inside the volume + args.push('/workspace/repo'); + + context.logger.info(`[OpenGrep Scanner] Command: opengrep ${args.join(' ')}`); + + const baseRunner = definition.runner; + if (baseRunner.kind !== 'docker') { + throw new ContainerError('OpenGrep runner must be docker', { + details: { reason: 'runner_type_mismatch', expected: 'docker', actual: baseRunner.kind }, + }); + } + + // Use the volume name passed from clone-repo's output + const volumeName = parsedInputs.volumeName; + + context.logger.info(`[OpenGrep Scanner] Mounting volume: ${volumeName}`); + + // Configure runner with volume mount + // Mount the clone-repo volume at /workspace; repo files are at /workspace/repo + const runnerConfig: DockerRunnerConfig = { + ...baseRunner, + command: args, + timeoutSeconds: parsedParams.timeout + 60, // Add buffer for startup + volumes: [ + { + source: volumeName, + target: '/workspace', + readOnly: true, + }, + ], + }; + + // Execute OpenGrep + const rawResult = await runComponentWithRunner( + runnerConfig, + async () => ({}) as Output, + { ...parsedInputs, ...parsedParams }, + context, + ); + + // Parse output — runner returns raw stdout string when output isn't valid JSON, + // or a parsed object when stdout was valid JSON (opengrep --json produces valid JSON) + let opengrepOutput: OpenGrepOutput; + let rawOutput: string; + + if (typeof rawResult === 'string') { + rawOutput = rawResult; + opengrepOutput = parseOpenGrepOutput(rawOutput); + } else if (rawResult && typeof rawResult === 'object') { + // Runner already parsed the JSON — use it directly + opengrepOutput = rawResult as OpenGrepOutput; + rawOutput = JSON.stringify(rawResult); + } else { + rawOutput = ''; + opengrepOutput = { results: [] }; + } + + const findings = convertToFindings(opengrepOutput.results || []); + + // Log any errors from OpenGrep + if (opengrepOutput.errors && opengrepOutput.errors.length > 0) { + for (const error of opengrepOutput.errors) { + context.logger.warn(`[OpenGrep Scanner] ${error.level}: ${error.message}`); + } + } + + context.logger.info(`[OpenGrep Scanner] Found ${findings.length} security issues`); + + // Count by severity + const severityCounts = findings.reduce( + (acc, f) => { + acc[f.severity] = (acc[f.severity] || 0) + 1; + return acc; + }, + {} as Record, + ); + + // Emit progress with results summary + if (findings.length > 0) { + const criticalHigh = (severityCounts.critical || 0) + (severityCounts.high || 0); + context.emitProgress({ + message: + criticalHigh > 0 + ? `⚠️ Found ${criticalHigh} critical/high severity issues` + : `Found ${findings.length} security issues`, + level: criticalHigh > 0 ? 'warn' : 'info', + data: { findingCount: findings.length, severityCounts }, + }); + } else { + context.emitProgress({ + message: 'No security issues detected', + level: 'info', + }); + } + + const output: Output = { + findings, + findingCount: findings.length, + rawOutput, + }; + + return outputSchema.parse(output); + }, +}); + +componentRegistry.register(definition); + +// Export types for external use +export type OpenGrepScannerInput = typeof inputSchema; +export type OpenGrepScannerOutput = typeof outputSchema; diff --git a/worker/src/components/scanners/trivy.ts b/worker/src/components/scanners/trivy.ts new file mode 100644 index 000000000..46713e04d --- /dev/null +++ b/worker/src/components/scanners/trivy.ts @@ -0,0 +1,484 @@ +import { z } from 'zod'; +import { + componentRegistry, + ComponentRetryPolicy, + runComponentWithRunner, + type DockerRunnerConfig, + ContainerError, + defineComponent, + inputs, + outputs, + parameters, + port, + param, +} from '@shipsec/component-sdk'; + +/** + * Trivy Scanner Component + * + * Scans container images, filesystems, and IaC for vulnerabilities. + * Designed to work with the GitHub clone-repo component via volumeName. + */ + +// Finding type for standardized output format +const findingSchema = z.object({ + type: z.string().describe('Type of finding (vulnerability, misconfiguration, secret)'), + target: z.string().describe('Target file or package'), + vulnerability_id: z.string().describe('CVE or misconfiguration ID'), + severity: z.enum(['critical', 'high', 'medium', 'low', 'info']).describe('Severity level'), + description: z.string().describe('Description of the finding'), +}); + +type Finding = z.infer; + +// Input schema - receives volumeName from clone component +const inputSchema = inputs({ + volumeName: port( + z + .string() + .min(1, 'Volume name is required') + .describe('Docker volume name containing the cloned repository'), + { + label: 'Volume Name', + description: + 'Docker volume name from github.repo.clone. The repository is at /repo inside the volume.', + connectionType: { kind: 'primitive', name: 'text' }, + }, + ), +}); + +// Configuration parameters +const parameterSchema = parameters({ + scanType: param( + z + .enum(['filesystem', 'config', 'image']) + .default('filesystem') + .describe('Type of scan to perform'), + { + label: 'Scan Type', + editor: 'select', + options: [ + { label: 'Filesystem (vulnerabilities in code/deps)', value: 'filesystem' }, + { label: 'Config (IaC misconfigurations)', value: 'config' }, + { label: 'Image (container image vulnerabilities)', value: 'image' }, + ], + description: 'Type of Trivy scan to perform.', + }, + ), + imageRef: param( + z.string().optional().describe('Container image reference (only for image scan type)'), + { + label: 'Image Reference', + editor: 'text', + placeholder: 'nginx:latest', + description: 'Container image to scan (only for image scan type).', + }, + ), + severityFilter: param( + z + .array(z.enum(['CRITICAL', 'HIGH', 'MEDIUM', 'LOW', 'UNKNOWN'])) + .optional() + .describe('Severity levels to include'), + { + label: 'Severity Filter', + editor: 'multi-select', + options: [ + { label: 'Critical', value: 'CRITICAL' }, + { label: 'High', value: 'HIGH' }, + { label: 'Medium', value: 'MEDIUM' }, + { label: 'Low', value: 'LOW' }, + { label: 'Unknown', value: 'UNKNOWN' }, + ], + description: 'Only include findings at these severity levels.', + }, + ), + scanners: param( + z + .array(z.enum(['vuln', 'misconfig', 'secret', 'license'])) + .optional() + .describe('Scanners to enable'), + { + label: 'Scanners', + editor: 'multi-select', + options: [ + { label: 'Vulnerabilities', value: 'vuln' }, + { label: 'Misconfigurations', value: 'misconfig' }, + { label: 'Secrets', value: 'secret' }, + { label: 'Licenses', value: 'license' }, + ], + description: 'Trivy scanners to enable. Leave empty for default (vuln, misconfig).', + }, + ), + ignoreUnfixed: param( + z.boolean().default(false).describe('Ignore vulnerabilities without fixed versions'), + { + label: 'Ignore Unfixed', + editor: 'boolean', + description: 'Skip vulnerabilities that do not have a fix available.', + }, + ), + timeout: param( + z.number().int().min(60).max(1800).default(300).describe('Scan timeout in seconds'), + { + label: 'Timeout (seconds)', + editor: 'number', + min: 60, + max: 1800, + description: 'Maximum time for the scan to complete.', + }, + ), +}); + +// Output schema - standardized findings format +const outputSchema = outputs({ + findings: port(z.array(findingSchema), { + label: 'Findings', + description: 'Array of detected issues in standardized format.', + connectionType: { kind: 'list', element: { kind: 'primitive', name: 'json' } }, + }), + findingCount: port(z.number(), { + label: 'Finding Count', + description: 'Total number of issues detected.', + }), + rawOutput: port(z.string(), { + label: 'Raw Output', + description: 'Raw Trivy JSON output for debugging.', + }), +}); + +type Output = z.infer; + +// Trivy JSON output format +interface TrivyVulnerability { + VulnerabilityID: string; + PkgName: string; + InstalledVersion: string; + FixedVersion?: string; + Severity: string; + Title?: string; + Description?: string; +} + +interface TrivyMisconfiguration { + ID: string; + Title: string; + Description: string; + Severity: string; + Resolution?: string; +} + +interface TrivyResult { + Target: string; + Type?: string; + Vulnerabilities?: TrivyVulnerability[]; + Misconfigurations?: TrivyMisconfiguration[]; + Secrets?: { + RuleID: string; + Category: string; + Severity: string; + Title: string; + Match: string; + }[]; +} + +interface TrivyOutput { + Results?: TrivyResult[]; +} + +// Map Trivy severity to standardized severity +function mapSeverity(trivySeverity: string): Finding['severity'] { + switch (trivySeverity.toUpperCase()) { + case 'CRITICAL': + return 'critical'; + case 'HIGH': + return 'high'; + case 'MEDIUM': + return 'medium'; + case 'LOW': + return 'low'; + default: + return 'info'; + } +} + +// Convert Trivy output to standardized findings +function convertToFindings(results: TrivyResult[]): Finding[] { + const findings: Finding[] = []; + + for (const result of results) { + // Process vulnerabilities + if (result.Vulnerabilities) { + for (const vuln of result.Vulnerabilities) { + findings.push({ + type: 'vulnerability', + target: `${result.Target} - ${vuln.PkgName}@${vuln.InstalledVersion}`, + vulnerability_id: vuln.VulnerabilityID, + severity: mapSeverity(vuln.Severity), + description: vuln.Description || vuln.Title || vuln.VulnerabilityID, + }); + } + } + + // Process misconfigurations + if (result.Misconfigurations) { + for (const misconfig of result.Misconfigurations) { + findings.push({ + type: 'misconfiguration', + target: result.Target, + vulnerability_id: misconfig.ID, + severity: mapSeverity(misconfig.Severity), + description: misconfig.Description || misconfig.Title, + }); + } + } + + // Process secrets + if (result.Secrets) { + for (const secret of result.Secrets) { + findings.push({ + type: 'secret', + target: result.Target, + vulnerability_id: secret.RuleID, + severity: mapSeverity(secret.Severity), + description: secret.Title, + }); + } + } + } + + return findings; +} + +// Parse Trivy JSON output +function parseTrivyOutput(rawOutput: string): TrivyOutput { + if (!rawOutput || rawOutput.trim().length === 0) { + return { Results: [] }; + } + + try { + return JSON.parse(rawOutput); + } catch { + // Try to find JSON in the output + const jsonMatch = rawOutput.match(/\{[\s\S]*"Results"[\s\S]*\}/); + if (jsonMatch) { + try { + return JSON.parse(jsonMatch[0]); + } catch { + return { Results: [] }; + } + } + return { Results: [] }; + } +} + +// Retry policy for Trivy +const trivyRetryPolicy: ComponentRetryPolicy = { + maxAttempts: 2, + initialIntervalSeconds: 10, + maximumIntervalSeconds: 60, + backoffCoefficient: 2, + nonRetryableErrorTypes: ['ContainerError', 'ValidationError', 'ConfigurationError'], +}; + +const definition = defineComponent({ + id: 'scanner.trivy', + label: 'Trivy Security Scanner', + category: 'scanners', + retryPolicy: trivyRetryPolicy, + runner: { + kind: 'docker', + image: 'aquasec/trivy:latest', + entrypoint: 'trivy', + network: 'bridge', // May need network for DB updates + timeoutSeconds: 600, + command: [], + env: { + HOME: '/tmp', + TRIVY_NO_PROGRESS: 'true', + }, + }, + inputs: inputSchema, + outputs: outputSchema, + parameters: parameterSchema, + docs: 'Scan filesystems, container images, and IaC for vulnerabilities and misconfigurations using Trivy.', + ui: { + slug: 'trivy-scanner', + version: '1.0.0', + type: 'scan', + category: 'scanners', + description: + 'Comprehensive vulnerability and misconfiguration scanner for filesystems, containers, and IaC.', + documentation: + 'Trivy is a comprehensive security scanner that detects vulnerabilities in OS packages, language packages, IaC misconfigurations, and secrets.', + documentationUrl: 'https://github.com/aquasecurity/trivy', + icon: 'ShieldAlert', + author: { + name: 'ShipSecAI', + type: 'shipsecai', + }, + isLatest: true, + deprecated: false, + example: + 'Clone a repository with github.repo.clone, then connect volumeName to this component for comprehensive security scanning.', + examples: [ + 'Scan filesystem for vulnerable packages.', + 'Check Terraform files for IaC misconfigurations.', + 'Detect hardcoded secrets in configuration files.', + 'Scan container images for vulnerabilities.', + ], + }, + async execute({ inputs, params }, context) { + const parsedInputs = inputSchema.parse(inputs); + const parsedParams = parameterSchema.parse(params); + + context.logger.info( + `[Trivy Scanner] Starting ${parsedParams.scanType} scan on volume: ${parsedInputs.volumeName}`, + ); + + context.emitProgress({ + message: `Launching Trivy ${parsedParams.scanType} scan...`, + level: 'info', + data: { volumeName: parsedInputs.volumeName, scanType: parsedParams.scanType }, + }); + + // Build Trivy command arguments + const args: string[] = [ + parsedParams.scanType === 'image' ? 'image' : parsedParams.scanType, + '--format', + 'json', + '--quiet', + ]; + + // Add severity filter + if (parsedParams.severityFilter && parsedParams.severityFilter.length > 0) { + args.push('--severity', parsedParams.severityFilter.join(',')); + } + + // Add scanners configuration + if (parsedParams.scanners && parsedParams.scanners.length > 0) { + args.push('--scanners', parsedParams.scanners.join(',')); + } + + // Add ignore unfixed flag + if (parsedParams.ignoreUnfixed) { + args.push('--ignore-unfixed'); + } + + // Add timeout + args.push('--timeout', `${parsedParams.timeout}s`); + + // Add target based on scan type + if (parsedParams.scanType === 'image') { + if (!parsedParams.imageRef) { + throw new ContainerError('Image reference is required for image scan type', { + details: { scanType: parsedParams.scanType }, + }); + } + args.push(parsedParams.imageRef); + } else { + args.push('/scan'); + } + + context.logger.info(`[Trivy Scanner] Command: trivy ${args.join(' ')}`); + + const baseRunner = definition.runner; + if (baseRunner.kind !== 'docker') { + throw new ContainerError('Trivy runner must be docker', { + details: { reason: 'runner_type_mismatch', expected: 'docker', actual: baseRunner.kind }, + }); + } + + // Configure runner with volume mount + const volumes: { source: string; target: string; readOnly: boolean }[] = []; + + // Only mount volume for filesystem/config scans + if (parsedParams.scanType !== 'image') { + volumes.push({ + source: parsedInputs.volumeName, + target: '/scan', + readOnly: true, + }); + } + + // For image scans, mount docker socket if available + if (parsedParams.scanType === 'image') { + volumes.push({ + source: '/var/run/docker.sock', + target: '/var/run/docker.sock', + readOnly: true, + }); + } + + const runnerConfig: DockerRunnerConfig = { + ...baseRunner, + command: args, + timeoutSeconds: parsedParams.timeout + 60, // Add buffer for startup + volumes: volumes.length > 0 ? volumes : undefined, + }; + + // Execute Trivy + const rawResult = await runComponentWithRunner( + runnerConfig, + async () => ({}) as Output, + { ...parsedInputs, ...parsedParams }, + context, + ); + + // Parse output + const rawOutput = typeof rawResult === 'string' ? rawResult : ''; + const trivyOutput = parseTrivyOutput(rawOutput); + const findings = convertToFindings(trivyOutput.Results || []); + + context.logger.info(`[Trivy Scanner] Found ${findings.length} issues`); + + // Count by severity + const severityCounts = findings.reduce( + (acc, f) => { + acc[f.severity] = (acc[f.severity] || 0) + 1; + return acc; + }, + {} as Record, + ); + + // Count by type + const typeCounts = findings.reduce( + (acc, f) => { + acc[f.type] = (acc[f.type] || 0) + 1; + return acc; + }, + {} as Record, + ); + + // Emit progress with results summary + if (findings.length > 0) { + const criticalHigh = (severityCounts.critical || 0) + (severityCounts.high || 0); + context.emitProgress({ + message: + criticalHigh > 0 + ? `⚠️ Found ${criticalHigh} critical/high severity issues` + : `Found ${findings.length} security issues`, + level: criticalHigh > 0 ? 'warn' : 'info', + data: { findingCount: findings.length, severityCounts, typeCounts }, + }); + } else { + context.emitProgress({ + message: 'No security issues detected', + level: 'info', + }); + } + + const output: Output = { + findings, + findingCount: findings.length, + rawOutput, + }; + + return outputSchema.parse(output); + }, +}); + +componentRegistry.register(definition); + +// Export types for external use +export type TrivyScannerInput = typeof inputSchema; +export type TrivyScannerOutput = typeof outputSchema; diff --git a/worker/src/components/scanners/trufflehog.ts b/worker/src/components/scanners/trufflehog.ts new file mode 100644 index 000000000..a254bfa1d --- /dev/null +++ b/worker/src/components/scanners/trufflehog.ts @@ -0,0 +1,488 @@ +import { z } from 'zod'; +import { + componentRegistry, + ComponentRetryPolicy, + runComponentWithRunner, + type DockerRunnerConfig, + ContainerError, + defineComponent, + inputs, + outputs, + parameters, + port, + param, +} from '@shipsec/component-sdk'; + +/** + * TruffleHog Secret Scanner Component + * + * Scans a cloned repository for exposed secrets and credentials. + * Designed to work with the GitHub clone-repo component via volumeName. + */ + +// Finding type for standardized output format +const findingSchema = z.object({ + type: z.string().describe('Type of finding (e.g., "AWS", "GitHub", "PrivateKey")'), + file: z.string().describe('File path where the secret was found'), + line: z.number().describe('Line number in the file'), + severity: z.enum(['critical', 'high', 'medium', 'low', 'info']).describe('Severity level'), + secret_type: z.string().describe('Specific type of secret detected'), + snippet: z.string().describe('Code snippet containing the finding (redacted)'), +}); + +type Finding = z.infer; + +// Input schema - receives volumeName from clone component +const inputSchema = inputs({ + volumeName: port( + z + .string() + .min(1, 'Volume name is required') + .describe('Docker volume name containing the cloned repository'), + { + label: 'Volume Name', + description: + 'Docker volume name from github.repo.clone. The repository is at /repo inside the volume.', + connectionType: { kind: 'primitive', name: 'text' }, + }, + ), + changedFiles: port( + z + .array(z.string()) + .optional() + .describe('List of changed file paths to scope the scan to (e.g. from PR context).'), + { + label: 'Changed Files', + description: + 'Optional list of file paths changed in the PR. When provided, only these files are scanned instead of the entire repository.', + connectionType: { kind: 'list', element: { kind: 'primitive', name: 'text' } }, + allowAny: true, + reason: 'Accepts any array of file path strings.', + }, + ), +}); + +// Configuration parameters +const parameterSchema = parameters({ + onlyVerified: param( + z.boolean().default(false).describe('Only report verified (actively valid) secrets'), + { + label: 'Only Verified', + editor: 'boolean', + description: 'Only report verified secrets that are confirmed to be active.', + helpText: + 'When enabled, passes --results=verified. When disabled, no --results flag is passed.', + }, + ), + includeDetectors: param( + z.array(z.string()).optional().describe('Specific detector types to include'), + { + label: 'Include Detectors', + editor: 'multi-select', + options: [ + { label: 'AWS', value: 'AWS' }, + { label: 'GitHub', value: 'Github' }, + { label: 'GitLab', value: 'Gitlab' }, + { label: 'Slack', value: 'Slack' }, + { label: 'Private Key', value: 'PrivateKey' }, + { label: 'Generic', value: 'GenericApiKey' }, + ], + description: 'Only run specific detectors (leave empty for all).', + }, + ), + excludeDetectors: param(z.array(z.string()).optional().describe('Detector types to exclude'), { + label: 'Exclude Detectors', + editor: 'multi-select', + options: [ + { label: 'AWS', value: 'AWS' }, + { label: 'GitHub', value: 'Github' }, + { label: 'GitLab', value: 'Gitlab' }, + { label: 'Slack', value: 'Slack' }, + { label: 'Private Key', value: 'PrivateKey' }, + { label: 'Generic', value: 'GenericApiKey' }, + ], + description: 'Exclude specific detectors from the scan.', + }), + maxDepth: param( + z.number().int().min(1).max(100).default(50).describe('Maximum git history depth to scan'), + { + label: 'Max Depth', + editor: 'number', + min: 1, + max: 100, + description: 'Maximum number of commits to scan in git history.', + }, + ), + concurrency: param( + z.number().int().min(1).max(20).default(8).describe('Number of concurrent workers'), + { + label: 'Concurrency', + editor: 'number', + min: 1, + max: 20, + description: 'Number of parallel scanning workers.', + }, + ), +}); + +// Output schema - standardized findings format +const outputSchema = outputs({ + findings: port(z.array(findingSchema), { + label: 'Findings', + description: 'Array of detected secrets in standardized format.', + connectionType: { kind: 'list', element: { kind: 'primitive', name: 'json' } }, + }), + findingCount: port(z.number(), { + label: 'Finding Count', + description: 'Total number of secrets detected.', + }), + verifiedCount: port(z.number(), { + label: 'Verified Count', + description: 'Number of verified (actively valid) secrets.', + }), + rawOutput: port(z.string(), { + label: 'Raw Output', + description: 'Raw TruffleHog JSON output for debugging.', + }), +}); + +type Output = z.infer; + +// TruffleHog raw output format +interface TruffleHogSecret { + DetectorType?: string | number; + DetectorName?: string; + Verified?: boolean; + Raw?: string; + Redacted?: string; + SourceMetadata?: { + Data?: { + Git?: { + commit?: string; + file?: string; + line?: number; + email?: string; + repository?: string; + timestamp?: string; + }; + Filesystem?: { + file?: string; + line?: number; + }; + }; + }; +} + +function toDetectorLabel(secret: TruffleHogSecret): string { + const detectorName = + typeof secret.DetectorName === 'string' ? secret.DetectorName.trim() : undefined; + if (detectorName && detectorName.length > 0) { + return detectorName; + } + + if (typeof secret.DetectorType === 'string') { + const detectorType = secret.DetectorType.trim(); + if (detectorType.length > 0) { + return detectorType; + } + } + + if (typeof secret.DetectorType === 'number') { + return String(secret.DetectorType); + } + + return 'Unknown'; +} + +// Map TruffleHog detector types to severity levels +function mapSeverity(detectorLabel: string, verified: boolean): Finding['severity'] { + // Verified secrets are always critical + if (verified) return 'critical'; + + // High-risk secret types + const normalized = detectorLabel.toLowerCase(); + const criticalTypes = ['aws', 'gcp', 'azure', 'privatekey', 'rsaprivatekey']; + const highTypes = ['github', 'gitlab', 'bitbucket', 'slack', 'stripe']; + const mediumTypes = ['genericapikey', 'genericpassword', 'jwt']; + + if (criticalTypes.some((t) => normalized.includes(t))) return 'high'; + if (highTypes.some((t) => normalized.includes(t))) return 'medium'; + if (mediumTypes.some((t) => normalized.includes(t))) return 'low'; + + return 'info'; +} + +// Known workspace prefixes that scanners add to file paths. +const WORKSPACE_PREFIXES = ['/workspace/repo/', '/scan/repo/', '/scan/', '/workspace/', '/repo/']; + +function stripWorkspacePrefix(filePath: string): string { + for (const prefix of WORKSPACE_PREFIXES) { + if (filePath.startsWith(prefix)) { + return filePath.slice(prefix.length); + } + } + if (filePath.startsWith('/')) { + return filePath.slice(1); + } + return filePath; +} + +// Convert TruffleHog output to standardized findings +function convertToFindings(secrets: TruffleHogSecret[]): Finding[] { + return secrets.map((secret) => { + const git = secret.SourceMetadata?.Data?.Git; + const fs = secret.SourceMetadata?.Data?.Filesystem; + const detectorLabel = toDetectorLabel(secret); + const rawFile = git?.file || fs?.file || 'unknown'; + + return { + type: detectorLabel, + file: stripWorkspacePrefix(rawFile), + line: git?.line || fs?.line || 0, + severity: mapSeverity(detectorLabel, secret.Verified || false), + secret_type: detectorLabel, + snippet: secret.Redacted || '[redacted]', + }; + }); +} + +function isTruffleHogSecret(value: unknown): value is TruffleHogSecret { + if (!value || typeof value !== 'object') { + return false; + } + + const candidate = value as Record; + return ( + 'DetectorName' in candidate || + 'DetectorType' in candidate || + ('SourceMetadata' in candidate && 'Raw' in candidate) + ); +} + +// Parse TruffleHog NDJSON output +function parseTruffleHogOutput(rawOutput: string): TruffleHogSecret[] { + if (!rawOutput || rawOutput.trim().length === 0) { + return []; + } + + const trimmed = rawOutput.trim(); + try { + const parsed = JSON.parse(trimmed); + if (Array.isArray(parsed)) { + return parsed.filter(isTruffleHogSecret); + } + if (isTruffleHogSecret(parsed)) { + return [parsed]; + } + } catch { + // Not a single JSON payload; continue with NDJSON parsing. + } + + const lines = rawOutput.split('\n').filter((line) => line.trim().length > 0); + const secrets: TruffleHogSecret[] = []; + + for (const line of lines) { + try { + const parsed = JSON.parse(line); + if (isTruffleHogSecret(parsed)) { + secrets.push(parsed); + } + } catch { + // Skip non-JSON lines (status messages, etc.) + continue; + } + } + + return secrets; +} + +// Retry policy for TruffleHog +const trufflehogRetryPolicy: ComponentRetryPolicy = { + maxAttempts: 2, + initialIntervalSeconds: 5, + maximumIntervalSeconds: 30, + backoffCoefficient: 2, + nonRetryableErrorTypes: ['ContainerError', 'ValidationError', 'ConfigurationError'], +}; + +const definition = defineComponent({ + id: 'scanner.trufflehog', + label: 'TruffleHog Secret Scanner', + category: 'scanners', + retryPolicy: trufflehogRetryPolicy, + runner: { + kind: 'docker', + image: 'trufflesecurity/trufflehog:latest', + // Shell wrapper pattern keeps PTY behavior stable and lets us merge stderr JSON into stdout. + entrypoint: 'sh', + network: 'none', // No network needed for filesystem scanning + timeoutSeconds: 600, + command: ['-c', 'trufflehog "$@" 2>&1', '--'], + env: { + HOME: '/tmp', + }, + }, + inputs: inputSchema, + outputs: outputSchema, + parameters: parameterSchema, + docs: 'Scan a cloned repository for exposed secrets and credentials using TruffleHog. Works with the github.repo.clone component.', + ui: { + slug: 'trufflehog-scanner', + version: '1.0.0', + type: 'scan', + category: 'scanners', + description: + 'Scan repositories for exposed secrets, API keys, and credentials using TruffleHog.', + documentation: + 'TruffleHog scans git history and files for 800+ credential types. Connect to the clone-repo component to scan GitHub repositories.', + documentationUrl: 'https://github.com/trufflesecurity/trufflehog', + icon: 'Key', + author: { + name: 'ShipSecAI', + type: 'shipsecai', + }, + isLatest: true, + deprecated: false, + example: + 'Clone a repository with github.repo.clone, then connect volumeName to this component.', + examples: [ + 'Scan a cloned repository for AWS credentials before deployment.', + 'Check for leaked API keys in PR changes.', + 'Audit repository history for accidentally committed secrets.', + ], + }, + async execute({ inputs, params }, context) { + const parsedInputs = inputSchema.parse(inputs); + const parsedParams = parameterSchema.parse(params); + + context.logger.info(`[TruffleHog Scanner] Starting scan on volume: ${parsedInputs.volumeName}`); + + context.emitProgress({ + message: 'Launching TruffleHog secret scan...', + level: 'info', + data: { volumeName: parsedInputs.volumeName }, + }); + + // Build TruffleHog command arguments + // Volume mounted at /scan, repo is at /scan/repo + const args: string[] = [ + 'filesystem', + '/scan/repo', + '--json', + '--no-update', // Don't auto-update + ]; + + // Add verification filter + if (parsedParams.onlyVerified) { + args.push('--results=verified'); + } + + // Add detector filters + if (parsedParams.includeDetectors && parsedParams.includeDetectors.length > 0) { + args.push('--include-detectors', parsedParams.includeDetectors.join(',')); + } + if (parsedParams.excludeDetectors && parsedParams.excludeDetectors.length > 0) { + args.push('--exclude-detectors', parsedParams.excludeDetectors.join(',')); + } + + // Add concurrency + args.push('--concurrency', parsedParams.concurrency.toString()); + + // Scope scan to changed files only (if provided) + // TruffleHog expects --include-paths pointing to a file with newline-separated regexes. + // We write the file to /tmp inside the container via the shell command. + const hasChangedFiles = parsedInputs.changedFiles && parsedInputs.changedFiles.length > 0; + if (hasChangedFiles) { + args.push('--include-paths', '/tmp/include-paths.txt'); + context.logger.info( + `[TruffleHog Scanner] Scoping to ${parsedInputs.changedFiles!.length} changed files`, + ); + } + + context.logger.info(`[TruffleHog Scanner] Command: trufflehog ${args.join(' ')}`); + + const baseRunner = definition.runner; + if (baseRunner.kind !== 'docker') { + throw new ContainerError('TruffleHog runner must be docker', { + details: { reason: 'runner_type_mismatch', expected: 'docker', actual: baseRunner.kind }, + }); + } + + // Build shell command — if changedFiles provided, write include-paths file first + let shellScript: string; + if (hasChangedFiles) { + // Escape single quotes in file paths for safe shell interpolation + const escapedPaths = parsedInputs.changedFiles!.map((f) => f.replace(/'/g, "'\\''")); + const writeLines = escapedPaths.map((f) => `echo '${f}'`).join(' && '); + shellScript = `(${writeLines}) > /tmp/include-paths.txt && trufflehog "$@" 2>&1`; + } else { + shellScript = 'trufflehog "$@" 2>&1'; + } + + // Configure runner with volume mount + const runnerConfig: DockerRunnerConfig = { + ...baseRunner, + command: ['-c', shellScript, '--', ...args], + volumes: [ + { + source: parsedInputs.volumeName, + target: '/scan', + readOnly: true, + }, + ], + }; + + // Execute TruffleHog + const rawResult = await runComponentWithRunner( + runnerConfig, + async () => ({}) as Output, + { ...parsedInputs, ...parsedParams }, + context, + ); + + // Parse output + const rawOutput = typeof rawResult === 'string' ? rawResult : ''; + const secrets = parseTruffleHogOutput(rawOutput); + const findings = convertToFindings(secrets); + const verifiedCount = secrets.filter((s) => s.Verified === true).length; + + context.logger.info( + `[TruffleHog Scanner] Found ${findings.length} secrets (${verifiedCount} verified)`, + ); + + // Emit progress with results summary + if (verifiedCount > 0) { + context.emitProgress({ + message: `⚠️ Found ${verifiedCount} verified secrets!`, + level: 'warn', + data: { findingCount: findings.length, verifiedCount }, + }); + } else if (findings.length > 0) { + context.emitProgress({ + message: `Found ${findings.length} potential secrets`, + level: 'info', + data: { findingCount: findings.length }, + }); + } else { + context.emitProgress({ + message: 'No secrets detected', + level: 'info', + }); + } + + const output: Output = { + findings, + findingCount: findings.length, + verifiedCount, + rawOutput, + }; + + return outputSchema.parse(output); + }, +}); + +componentRegistry.register(definition); + +// Export types for external use +export type TruffleHogScannerInput = typeof inputSchema; +export type TruffleHogScannerOutput = typeof outputSchema; diff --git a/worker/src/components/security/__tests__/findings-normalize-filter.test.ts b/worker/src/components/security/__tests__/findings-normalize-filter.test.ts new file mode 100644 index 000000000..ddd98c209 --- /dev/null +++ b/worker/src/components/security/__tests__/findings-normalize-filter.test.ts @@ -0,0 +1,49 @@ +import { describe, expect, it } from 'bun:test'; +import { stripWorkspacePrefix } from '../findings-normalize'; + +describe('stripWorkspacePrefix', () => { + it('should strip /workspace/repo/ prefix', () => { + expect(stripWorkspacePrefix('/workspace/repo/src/main.ts')).toBe('src/main.ts'); + }); + + it('should strip /scan/repo/ prefix', () => { + expect(stripWorkspacePrefix('/scan/repo/src/main.ts')).toBe('src/main.ts'); + }); + + it('should strip /scan/ prefix', () => { + expect(stripWorkspacePrefix('/scan/src/main.ts')).toBe('src/main.ts'); + }); + + it('should strip /workspace/ prefix', () => { + expect(stripWorkspacePrefix('/workspace/src/main.ts')).toBe('src/main.ts'); + }); + + it('should strip /repo/ prefix', () => { + expect(stripWorkspacePrefix('/repo/src/main.ts')).toBe('src/main.ts'); + }); + + it('should strip leading slash when no known prefix matches', () => { + expect(stripWorkspacePrefix('/src/main.ts')).toBe('src/main.ts'); + }); + + it('should return relative paths unchanged', () => { + expect(stripWorkspacePrefix('src/main.ts')).toBe('src/main.ts'); + }); + + it('should handle empty string', () => { + expect(stripWorkspacePrefix('')).toBe(''); + }); + + it('should prefer longest matching prefix', () => { + // /workspace/repo/ is checked before /workspace/ due to array order + expect(stripWorkspacePrefix('/workspace/repo/file.ts')).toBe('file.ts'); + }); + + it('should handle file at root level', () => { + expect(stripWorkspacePrefix('/workspace/repo/package.json')).toBe('package.json'); + }); + + it('should handle deeply nested paths', () => { + expect(stripWorkspacePrefix('/workspace/repo/a/b/c/d/e.ts')).toBe('a/b/c/d/e.ts'); + }); +}); diff --git a/worker/src/components/security/__tests__/trufflehog.test.ts b/worker/src/components/security/__tests__/trufflehog.test.ts index be7a5a354..211e7e16a 100644 --- a/worker/src/components/security/__tests__/trufflehog.test.ts +++ b/worker/src/components/security/__tests__/trufflehog.test.ts @@ -283,6 +283,234 @@ describe('trufflehog component', () => { expect(result.results).toHaveLength(2); }); + it('should not pass --results flag when onlyVerified is false', async () => { + const component = componentRegistry.get( + 'shipsec.trufflehog.scan', + ); + if (!component) throw new Error('Component not registered'); + + const context = sdk.createExecutionContext({ + runId: 'test-run', + componentRef: 'trufflehog-test', + }); + + const executePayload = { + inputs: { + scanTarget: 'https://github.com/test/repo', + }, + params: { + scanType: 'git' as const, + onlyVerified: false, + }, + }; + + const spy = vi.spyOn(sdk, 'runComponentWithRunner').mockResolvedValue( + JSON.stringify({ + secrets: [], + rawOutput: '', + secretCount: 0, + verifiedCount: 0, + hasVerifiedSecrets: false, + }), + ); + + await component.execute(executePayload, context); + + const runnerConfig = spy.mock.calls[0]?.[0] as { command?: string[] }; + const command = runnerConfig.command ?? []; + expect(command.some((arg) => arg.startsWith('--results='))).toBe(false); + }); + + it('should pass --results=verified when onlyVerified is true', async () => { + const component = componentRegistry.get( + 'shipsec.trufflehog.scan', + ); + if (!component) throw new Error('Component not registered'); + + const context = sdk.createExecutionContext({ + runId: 'test-run', + componentRef: 'trufflehog-test', + }); + + const executePayload = { + inputs: { + scanTarget: 'https://github.com/test/repo', + }, + params: { + scanType: 'git' as const, + onlyVerified: true, + }, + }; + + const spy = vi.spyOn(sdk, 'runComponentWithRunner').mockResolvedValue( + JSON.stringify({ + secrets: [], + rawOutput: '', + secretCount: 0, + verifiedCount: 0, + hasVerifiedSecrets: false, + }), + ); + + await component.execute(executePayload, context); + + const runnerConfig = spy.mock.calls[0]?.[0] as { command?: string[] }; + const command = runnerConfig.command ?? []; + expect(command).toContain('--results=verified'); + }); + + it('should not pass --results flag when onlyVerified is false', async () => { + const component = componentRegistry.get( + 'shipsec.trufflehog.scan', + ); + if (!component) throw new Error('Component not registered'); + + const context = sdk.createExecutionContext({ + runId: 'test-run', + componentRef: 'trufflehog-test', + }); + + const executePayload = { + inputs: { + scanTarget: 'https://github.com/test/repo', + }, + params: { + scanType: 'git' as const, + onlyVerified: false, + }, + }; + + const spy = vi.spyOn(sdk, 'runComponentWithRunner').mockResolvedValue( + JSON.stringify({ + secrets: [], + rawOutput: '', + secretCount: 0, + verifiedCount: 0, + hasVerifiedSecrets: false, + }), + ); + + await component.execute(executePayload, context); + + const runnerConfig = spy.mock.calls[0]?.[0] as { command?: string[] }; + const command = runnerConfig.command ?? []; + expect(command.some((arg) => arg.startsWith('--results='))).toBe(false); + }); + + it('should pass --results=verified when onlyVerified is true', async () => { + const component = componentRegistry.get( + 'shipsec.trufflehog.scan', + ); + if (!component) throw new Error('Component not registered'); + + const context = sdk.createExecutionContext({ + runId: 'test-run', + componentRef: 'trufflehog-test', + }); + + const executePayload = { + inputs: { + scanTarget: 'https://github.com/test/repo', + }, + params: { + scanType: 'git' as const, + onlyVerified: true, + }, + }; + + const spy = vi.spyOn(sdk, 'runComponentWithRunner').mockResolvedValue( + JSON.stringify({ + secrets: [], + rawOutput: '', + secretCount: 0, + verifiedCount: 0, + hasVerifiedSecrets: false, + }), + ); + + await component.execute(executePayload, context); + + const runnerConfig = spy.mock.calls[0]?.[0] as { command?: string[] }; + const command = runnerConfig.command ?? []; + expect(command).toContain('--results=verified'); + }); + + it('should not pass --results flag when onlyVerified is false', async () => { + const component = componentRegistry.get( + 'shipsec.trufflehog.scan', + ); + if (!component) throw new Error('Component not registered'); + + const context = sdk.createExecutionContext({ + runId: 'test-run', + componentRef: 'trufflehog-test', + }); + + const executePayload = { + inputs: { + scanTarget: 'https://github.com/test/repo', + }, + params: { + scanType: 'git' as const, + onlyVerified: false, + }, + }; + + const spy = vi.spyOn(sdk, 'runComponentWithRunner').mockResolvedValue( + JSON.stringify({ + secrets: [], + rawOutput: '', + secretCount: 0, + verifiedCount: 0, + hasVerifiedSecrets: false, + }), + ); + + await component.execute(executePayload, context); + + const runnerConfig = spy.mock.calls[0]?.[0] as { command?: string[] }; + const command = runnerConfig.command ?? []; + expect(command.some((arg) => arg.startsWith('--results='))).toBe(false); + }); + + it('should pass --results=verified when onlyVerified is true', async () => { + const component = componentRegistry.get( + 'shipsec.trufflehog.scan', + ); + if (!component) throw new Error('Component not registered'); + + const context = sdk.createExecutionContext({ + runId: 'test-run', + componentRef: 'trufflehog-test', + }); + + const executePayload = { + inputs: { + scanTarget: 'https://github.com/test/repo', + }, + params: { + scanType: 'git' as const, + onlyVerified: true, + }, + }; + + const spy = vi.spyOn(sdk, 'runComponentWithRunner').mockResolvedValue( + JSON.stringify({ + secrets: [], + rawOutput: '', + secretCount: 0, + verifiedCount: 0, + hasVerifiedSecrets: false, + }), + ); + + await component.execute(executePayload, context); + + const runnerConfig = spy.mock.calls[0]?.[0] as { command?: string[] }; + const command = runnerConfig.command ?? []; + expect(command).toContain('--results=verified'); + }); + it('should handle parse errors gracefully', async () => { const component = componentRegistry.get( 'shipsec.trufflehog.scan', @@ -381,8 +609,8 @@ describe('trufflehog component', () => { }, }; - const error = new Error('Container exited with code 183'); - vi.spyOn(sdk, 'runComponentWithRunner').mockRejectedValue(error); + const error = { message: 'Container exited with code 183' }; + vi.spyOn(sdk, 'runComponentWithRunner').mockRejectedValue(error as any); await expect(component.execute(executePayload, context)).rejects.toThrow( 'Container exited with code 183', @@ -409,8 +637,8 @@ describe('trufflehog component', () => { }, }; - const error = new Error('Container exited with code 1: auth failed'); - vi.spyOn(sdk, 'runComponentWithRunner').mockRejectedValue(error); + const error = { message: 'Container exited with code 1: auth failed' }; + vi.spyOn(sdk, 'runComponentWithRunner').mockRejectedValue(error as any); await expect(component.execute(executePayload, context)).rejects.toThrow('auth failed'); }); diff --git a/worker/src/components/security/findings-markdown.ts b/worker/src/components/security/findings-markdown.ts new file mode 100644 index 000000000..0e3103af7 --- /dev/null +++ b/worker/src/components/security/findings-markdown.ts @@ -0,0 +1,332 @@ +import { z } from 'zod'; +import { + componentRegistry, + defineComponent, + inputs, + outputs, + parameters, + port, + param, +} from '@shipsec/component-sdk'; + +const LOG_PREFIX = '[Findings Markdown]'; + +/** + * Findings Markdown Formatter Component + * + * Converts normalized findings into a Markdown report suitable for PR comments. + * Features: + * - Summary table by severity + * - Collapsible details for each finding + * - Grouping by scanner + * - Truncation for large reports (GitHub comment size limits) + */ + +// Normalized finding schema (from security.findings.normalize) +const severitySchema = z.enum(['critical', 'high', 'medium', 'low', 'info', 'none']); + +const normalizedFindingSchema = z.object({ + scanner: z.string(), + finding_hash: z.string(), + severity: severitySchema, + asset_key: z.string().optional(), + rule_id: z.string(), + title: z.string(), + description: z.string().optional(), + file: z.string().optional(), + line: z.number().optional(), + snippet: z.string().optional(), + metadata: z.record(z.string(), z.unknown()).optional(), +}); + +const inputSchema = inputs({ + findings: port(z.array(normalizedFindingSchema), { + label: 'Normalized Findings', + description: 'Array of findings from security.findings.normalize', + connectionType: { kind: 'list', element: { kind: 'primitive', name: 'json' } }, + }), + prContext: port( + z + .object({ + owner: z.string(), + repo: z.string(), + pullNumber: z.number().optional(), + headSha: z.string().optional(), + cloneUrl: z.string().optional(), + }) + .optional(), + { + label: 'PR Context', + description: 'Optional PR context to add links to files', + connectionType: { kind: 'any' }, + }, + ), +}); + +export type FindingsMarkdownInput = typeof inputSchema; + +const outputSchema = outputs({ + markdown: port(z.string(), { + label: 'Markdown Report', + description: 'Formatted Markdown string ready for PR comment', + connectionType: { kind: 'primitive', name: 'text' }, + }), + hasFindings: port(z.boolean(), { + label: 'Has Findings', + description: 'True if there are any findings in the report', + connectionType: { kind: 'primitive', name: 'boolean' }, + }), +}); + +export type FindingsMarkdownOutput = typeof outputSchema; + +const parameterSchema = parameters({ + title: param(z.string().default('Security Scan Results'), { + label: 'Report Title', + editor: 'text', + description: 'Title header for the notification', + }), + grouping: param(z.enum(['scanner', 'severity', 'none']).default('scanner'), { + label: 'Group By', + editor: 'select', + options: [ + { label: 'Scanner', value: 'scanner' }, + { label: 'Severity', value: 'severity' }, + { label: 'None (Flat List)', value: 'none' }, + ], + description: 'How to group findings in the report', + }), + maxFindings: param(z.number().min(1).max(50).default(20), { + label: 'Max Findings', + editor: 'number', + description: 'Maximum number of findings to show in detail before truncating', + }), + showScanner: param(z.boolean().default(false), { + label: 'Show Scanner Tag', + editor: 'boolean', + description: + 'Append the scanner name (e.g. opengrep) to each finding line. Useful when grouping by severity across multiple scanners.', + }), + logoUrl: param( + z.string().optional().default('https://avatars.githubusercontent.com/u/211031279?s=200&v=4'), + { + label: 'Logo URL', + editor: 'text', + description: 'URL of the logo image to display in the footer.', + }, + ), +}); + +// Helper: Get emoji for severity +function getSeverityEmoji(severity: string): string { + switch (severity.toLowerCase()) { + case 'critical': + return '🔴'; + case 'high': + return '🟠'; + case 'medium': + return '🟡'; + case 'low': + return '🔵'; + case 'info': + return 'ℹ️'; + default: + return '⚪'; + } +} + +// Helper: Generate file link +function getFileLink( + file: string, + line: number | undefined, + context: z.infer['prContext'], +): string { + if (!context || !context.owner || !context.repo || !context.headSha) { + return line ? `${file}:${line}` : file; + } + // GitHub permalink + const url = `https://github.com/${context.owner}/${context.repo}/blob/${context.headSha}/${file}${line ? `#L${line}` : ''}`; + return `[${file}${line ? `:${line}` : ''}](${url})`; +} + +const definition = defineComponent({ + id: 'security.findings.markdown', + label: 'Format Findings to Markdown', + category: 'notification', + runner: { kind: 'inline' }, + inputs: inputSchema, + outputs: outputSchema, + parameters: parameterSchema, + docs: 'Converts normalized findings into a rich Markdown report with severity summaries, file links, and categorization.', + ui: { + slug: 'security-findings-markdown', + version: '1.0.0', + type: 'output', + category: 'notification', + description: 'Format security findings as a Markdown report for PR comments.', + documentation: + 'Connect normalized findings to generate a GitHub-flavored Markdown report. Supports grouping by scanner or severity, and limits output size for comments.', + icon: 'FileText', + author: { + name: 'ShipSecAI', + type: 'shipsecai', + }, + isLatest: true, + deprecated: false, + example: 'Normalize Findings → Format Findings → Post PR Comment', + examples: ['Create a daily security report', 'Generate PR comment body from multiple scanners'], + }, + async execute({ inputs, params }, context) { + const { findings, prContext } = inputSchema.parse(inputs); + const { title, grouping, maxFindings, showScanner, logoUrl } = parameterSchema.parse(params); + + context.logger.info(`${LOG_PREFIX} Formatting ${findings.length} findings into Markdown`); + + // Build context line for header + const contextParts: string[] = []; + if (prContext?.owner && prContext?.repo) { + contextParts.push(`\`${prContext.owner}/${prContext.repo}\``); + } + if (prContext?.pullNumber) { + contextParts.push(`PR #${prContext.pullNumber}`); + } + if (prContext?.headSha) { + contextParts.push(`\`${prContext.headSha.slice(0, 7)}\``); + } + const contextLine = contextParts.length > 0 ? contextParts.join(' | ') : ''; + + if (findings.length === 0) { + let emptyReport = `## ${title}\n\n`; + if (contextLine) emptyReport += `${contextLine}\n\n`; + const logoTag = logoUrl + ? ` ` + : ''; + emptyReport += `> **No issues found**\n\n---\n${logoTag}ShipSec`; + return outputSchema.parse({ + markdown: emptyReport, + hasFindings: false, + }); + } + + // summary stats + const stats = { + critical: findings.filter((f) => f.severity === 'critical').length, + high: findings.filter((f) => f.severity === 'high').length, + medium: findings.filter((f) => f.severity === 'medium').length, + low: findings.filter((f) => f.severity === 'low').length, + info: findings.filter((f) => f.severity === 'info' || f.severity === 'none').length, + total: findings.length, + }; + + let md = `## ${title}\n\n`; + if (contextLine) md += `${contextLine}\n\n`; + + // 1. Compact severity badges + const badges: string[] = []; + if (stats.critical > 0) badges.push(`🔴 **${stats.critical}** critical`); + if (stats.high > 0) badges.push(`🟠 **${stats.high}** high`); + if (stats.medium > 0) badges.push(`🟡 **${stats.medium}** medium`); + if (stats.low > 0) badges.push(`🔵 **${stats.low}** low`); + if (stats.info > 0) badges.push(`ℹ️ **${stats.info}** info`); + md += `> **${stats.total} finding${stats.total !== 1 ? 's' : ''}:** ${badges.join(' · ')}\n\n`; + + // 2. Truncation Warning + const shownFindings = findings.slice(0, maxFindings); + const hiddenCount = findings.length - maxFindings; + + if (hiddenCount > 0) { + md += `> Showing ${maxFindings} of ${findings.length} findings.\n\n`; + } + + // Helper to truncate long descriptions + const truncateDesc = (desc: string, max = 200): string => { + if (desc.length <= max) return desc; + return desc.slice(0, max).replace(/\s+\S*$/, '') + '...'; + }; + + // Helper to render a finding + const renderFinding = (f: z.infer) => { + const icon = getSeverityEmoji(f.severity); + const fileLink = f.file ? getFileLink(f.file, f.line, prContext) : ''; + const scanner = showScanner ? ` _(${f.scanner})_` : ''; + + let entry = `
\n${icon} ${f.title}`; + if (fileLink) entry += ` — ${fileLink}`; + entry += `${scanner}\n\n`; + + // Rule ID as inline code + entry += `**Rule:** \`${f.rule_id}\`\n\n`; + + if (f.description) { + entry += `${truncateDesc(f.description)}\n\n`; + } + + if (f.snippet) { + const ext = f.file?.split('.').pop() || ''; + const lang = /^[a-z0-9]+$/i.test(ext) ? ext : ''; + entry += `\`\`\`${lang}\n${f.snippet.trim()}\n\`\`\`\n`; + } + + entry += `\n
\n\n`; + return entry; + }; + + if (grouping === 'scanner') { + const byScanner: Record = {}; + shownFindings.forEach((f) => { + const s = f.scanner || 'unknown'; + if (!byScanner[s]) byScanner[s] = []; + byScanner[s].push(f); + }); + + Object.keys(byScanner) + .sort() + .forEach((scanner) => { + md += `### ${scanner.toUpperCase()}\n\n`; + byScanner[scanner].forEach((f) => { + md += renderFinding(f); + }); + }); + } else if (grouping === 'severity') { + const order = ['critical', 'high', 'medium', 'low', 'info', 'none']; + const bySeverity: Record = {}; + shownFindings.forEach((f) => { + const s = f.severity; + if (!bySeverity[s]) bySeverity[s] = []; + bySeverity[s].push(f); + }); + + order.forEach((sev) => { + if (bySeverity[sev] && bySeverity[sev].length > 0) { + md += `### ${getSeverityEmoji(sev)} ${sev.charAt(0).toUpperCase() + sev.slice(1)}\n\n`; + bySeverity[sev].forEach((f) => { + md += renderFinding(f); + }); + } + }); + } else { + shownFindings.forEach((f) => { + md += renderFinding(f); + }); + } + + if (logoUrl) { + md += `---\n ShipSec`; + } else { + md += `---\nShipSec`; + } + + context.emitProgress({ + message: `Formatted ${findings.length} findings into Markdown`, + level: 'info', + data: { stats, length: md.length }, + }); + + return outputSchema.parse({ + markdown: md, + hasFindings: true, + }); + }, +}); + +componentRegistry.register(definition); diff --git a/worker/src/components/security/findings-normalize.ts b/worker/src/components/security/findings-normalize.ts new file mode 100644 index 000000000..3c3263c5a --- /dev/null +++ b/worker/src/components/security/findings-normalize.ts @@ -0,0 +1,611 @@ +import { z } from 'zod'; +import { createHash } from 'crypto'; +import { + componentRegistry, + defineComponent, + inputs, + outputs, + parameters, + port, + param, +} from '@shipsec/component-sdk'; + +const LOG_PREFIX = '[Findings Normalize]'; + +/** + * Security Findings Normalizer Component + * + * Converts raw scanner output into standardized AnalyticsResult format. + * Supports multiple scanner types: TruffleHog, OpenGrep, Dependency, Trivy. + * + * This enables: + * - Unified PR comments with findings from multiple scanners + * - Consistent OpenSearch indexing via Analytics Sink + * - Filtering findings to only changed files in PRs + * - Cross-scanner deduplication via finding_hash + */ + +// Severity enum matching analytics schema +const severitySchema = z.enum(['critical', 'high', 'medium', 'low', 'info', 'none']); +type Severity = z.infer; + +// Normalized output format compatible with Analytics Sink +const normalizedFindingSchema = z.object({ + scanner: z.string().describe('Scanner that produced this finding'), + finding_hash: z.string().describe('Stable 16-char hash for deduplication'), + severity: severitySchema.describe('Normalized severity level'), + asset_key: z.string().optional().describe('Primary asset identifier (usually file path)'), + // Common fields + rule_id: z.string().describe('Rule or detector ID'), + title: z.string().describe('Short title/summary of the finding'), + description: z.string().optional().describe('Detailed description'), + file: z.string().optional().describe('File path'), + line: z.number().optional().describe('Line number'), + snippet: z.string().optional().describe('Code snippet'), + // Tool-specific fields preserved + metadata: z.record(z.string(), z.unknown()).optional().describe('Tool-specific metadata'), +}); + +type NormalizedFinding = z.infer; + +// Summary output +const summarySchema = z.object({ + critical: z.number(), + high: z.number(), + medium: z.number(), + low: z.number(), + info: z.number(), + total: z.number(), +}); + +// Input: Raw findings from any scanner (accepts any JSON structure) +const inputSchema = inputs({ + rawFindings: port(z.array(z.record(z.string(), z.unknown())), { + label: 'Raw Findings', + description: 'Array of findings from a scanner component', + connectionType: { kind: 'list', element: { kind: 'primitive', name: 'json' } }, + }), + changedFiles: port(z.array(z.string()).optional(), { + label: 'Changed Files', + description: 'Optional list of changed files (from github.pr.context) to filter findings', + connectionType: { kind: 'list', element: { kind: 'primitive', name: 'text' } }, + }), +}); + +export type FindingsNormalizeInput = typeof inputSchema; + +const outputSchema = outputs({ + findings: port(z.array(normalizedFindingSchema), { + label: 'Normalized Findings', + description: 'Findings in standardized format compatible with Analytics Sink', + connectionType: { kind: 'list', element: { kind: 'primitive', name: 'json' } }, + }), + summary: port(summarySchema, { + label: 'Summary', + description: 'Count of findings by severity', + connectionType: { kind: 'any' }, + }), + findingCount: port(z.number(), { + label: 'Finding Count', + description: 'Total number of findings after filtering', + connectionType: { kind: 'primitive', name: 'number' }, + }), + filteredCount: port(z.number(), { + label: 'Filtered Count', + description: 'Number of findings filtered out (not in changed files)', + connectionType: { kind: 'primitive', name: 'number' }, + }), +}); + +export type FindingsNormalizeOutput = typeof outputSchema; + +const parameterSchema = parameters({ + scannerType: param( + z + .enum(['trufflehog', 'opengrep', 'dependency', 'trivy', 'nuclei', 'prowler', 'auto']) + .default('auto'), + { + label: 'Scanner Type', + editor: 'select', + options: [ + { label: 'Auto-detect', value: 'auto' }, + { label: 'TruffleHog (Secrets)', value: 'trufflehog' }, + { label: 'OpenGrep (SAST)', value: 'opengrep' }, + { label: 'Dependency Scanner', value: 'dependency' }, + { label: 'Trivy (Container/IaC)', value: 'trivy' }, + { label: 'Nuclei', value: 'nuclei' }, + { label: 'Prowler', value: 'prowler' }, + ], + description: 'Type of scanner that produced the findings (auto-detect if unsure)', + }, + ), + filterToChangedFiles: param(z.boolean().default(true), { + label: 'Filter to Changed Files', + editor: 'boolean', + description: 'Only include findings in files from changedFiles input (for PR scans)', + }), +}); + +/** + * Generate a stable hash for finding deduplication. + * Uses SHA-256 truncated to 16 chars. + */ +function generateFindingHash(...fields: (string | number | undefined | null)[]): string { + const normalized = fields.map((f) => (f == null ? '' : String(f).toLowerCase().trim())).join('|'); + return createHash('sha256').update(normalized).digest('hex').slice(0, 16); +} + +/** + * Known workspace prefixes that scanners add to file paths. + * Strip these before comparing against GitHub's relative paths. + */ +const WORKSPACE_PREFIXES = ['/workspace/repo/', '/scan/repo/', '/scan/', '/workspace/', '/repo/']; + +export function stripWorkspacePrefix(filePath: string): string { + for (const prefix of WORKSPACE_PREFIXES) { + if (filePath.startsWith(prefix)) { + return filePath.slice(prefix.length); + } + } + // Strip leading slash if present + if (filePath.startsWith('/')) { + return filePath.slice(1); + } + return filePath; +} + +/** + * Detect scanner type from finding structure + */ +function detectScannerType(finding: Record): string { + // TruffleHog: has type, secret_type, snippet + if ('type' in finding && 'secret_type' in finding) { + return 'trufflehog'; + } + // OpenGrep: has rule_id, message, severity + if ('rule_id' in finding && 'message' in finding) { + return 'opengrep'; + } + // Dependency: has package, vulnerability_id + if ('package' in finding && ('vulnerability_id' in finding || 'vuln_id' in finding)) { + return 'dependency'; + } + // Trivy: has VulnerabilityID, PkgName + if ('VulnerabilityID' in finding || 'Target' in finding) { + return 'trivy'; + } + // Nuclei: has template-id, matched-at + if ('template-id' in finding || 'templateID' in finding) { + return 'nuclei'; + } + // Prowler: has check_id, status + if ('check_id' in finding && 'status' in finding) { + return 'prowler'; + } + return 'unknown'; +} + +/** + * Normalize TruffleHog finding + */ +function normalizeTruffleHog(finding: Record): NormalizedFinding { + const type = String(finding.type || 'secret'); + const rawFile = String(finding.file || ''); + const file = stripWorkspacePrefix(rawFile); + const line = Number(finding.line || 0); + const secretType = String(finding.secret_type || type); + const severity = (finding.severity as Severity) || 'high'; + const snippet = String(finding.snippet || ''); + const verified = finding.verified === true; + + return { + scanner: 'trufflehog', + finding_hash: generateFindingHash('trufflehog', type, rawFile, line, secretType), + severity, + asset_key: file, + rule_id: secretType, + title: `${secretType} secret${verified ? ' (verified)' : ''}`, + description: `Detected ${secretType} secret in \`${file}\`${verified ? ' — **verified active**' : ''}`, + file, + line, + snippet, + metadata: { + detectorType: type, + secretType, + verified, + }, + }; +} + +/** + * Normalize OpenGrep finding + */ +/** + * Extract a human-readable title from an OpenGrep rule ID. + * e.g. "python.flask.security.injection.sql-injection.tainted-sql" → "SQL Injection — Tainted SQL" + * e.g. "generic.secrets.security.detected-stripe-api-key.detected-stripe-api-key" → "Detected Stripe API Key" + */ +function openGrepRuleTitle(ruleId: string): string { + // Take the last segment (most specific), fall back to second-to-last if they're identical + const parts = ruleId.split('.'); + let slug = parts[parts.length - 1] || ruleId; + // If last two segments are identical (common pattern), use second-to-last + if (parts.length >= 2 && parts[parts.length - 1] === parts[parts.length - 2]) { + slug = parts[parts.length - 1]; + } + // Common acronym map for proper capitalization + const acronyms: Record = { + sql: 'SQL', + api: 'API', + xss: 'XSS', + html: 'HTML', + css: 'CSS', + xml: 'XML', + jwt: 'JWT', + url: 'URL', + http: 'HTTP', + https: 'HTTPS', + ssh: 'SSH', + ssl: 'SSL', + tls: 'TLS', + rsa: 'RSA', + aws: 'AWS', + gcp: 'GCP', + dns: 'DNS', + ip: 'IP', + os: 'OS', + db: 'DB', + iam: 'IAM', + rce: 'RCE', + ssrf: 'SSRF', + csrf: 'CSRF', + ldap: 'LDAP', + oidc: 'OIDC', + saml: 'SAML', + cors: 'CORS', + sast: 'SAST', + }; + // Convert kebab-case to Title Case, then fix acronyms + return slug + .replace(/-/g, ' ') + .replace(/_/g, ' ') + .replace(/\b\w/g, (c) => c.toUpperCase()) + .replace(/\b\w+\b/g, (word) => acronyms[word.toLowerCase()] || word); +} + +function normalizeOpenGrep(finding: Record): NormalizedFinding { + const ruleId = String(finding.rule_id || 'unknown'); + const rawFile = String(finding.file || ''); + const file = stripWorkspacePrefix(rawFile); + const line = Number(finding.line || 0); + const message = String(finding.message || ''); + const severity = (finding.severity as Severity) || 'medium'; + const snippet = String(finding.snippet || ''); + + return { + scanner: 'opengrep', + finding_hash: generateFindingHash('opengrep', ruleId, rawFile, line), + severity, + asset_key: file, + rule_id: ruleId, + title: openGrepRuleTitle(ruleId), + description: message, + file, + line, + snippet, + metadata: { + checkId: ruleId, + }, + }; +} + +/** + * Normalize Dependency finding + */ +function normalizeDependency(finding: Record): NormalizedFinding { + const pkg = String(finding.package || finding.Package || ''); + const version = String(finding.version || finding.Version || ''); + const vulnId = String( + finding.vulnerability_id || finding.vuln_id || finding.VulnerabilityID || '', + ); + const severity = mapDependencySeverity(String(finding.severity || finding.Severity || 'unknown')); + const fixedVersion = String(finding.fixed_version || finding.FixedVersion || ''); + + return { + scanner: 'dependency', + finding_hash: generateFindingHash('dependency', pkg, version, vulnId), + severity, + asset_key: `${pkg}@${version}`, + rule_id: vulnId, + title: `${vulnId} in ${pkg}`, + description: `Vulnerable dependency: ${pkg}@${version}. ${fixedVersion ? `Fixed in: ${fixedVersion}` : 'No fix available'}`, + metadata: { + package: pkg, + version, + vulnerabilityId: vulnId, + fixedVersion: fixedVersion || undefined, + }, + }; +} + +/** + * Normalize Trivy finding + */ +function normalizeTrivy(finding: Record): NormalizedFinding { + const vulnId = String(finding.VulnerabilityID || finding.vulnerability_id || ''); + const pkgName = String(finding.PkgName || finding.package || ''); + const target = String(finding.Target || finding.target || ''); + const severity = mapTrivySeverity(String(finding.Severity || finding.severity || 'unknown')); + const title = String(finding.Title || finding.title || vulnId); + const description = String(finding.Description || finding.description || ''); + + return { + scanner: 'trivy', + finding_hash: generateFindingHash('trivy', target, pkgName, vulnId), + severity, + asset_key: target || pkgName, + rule_id: vulnId, + title, + description, + file: target, + metadata: { + vulnerabilityId: vulnId, + packageName: pkgName, + target, + }, + }; +} + +/** + * Normalize Nuclei finding + */ +function normalizeNuclei(finding: Record): NormalizedFinding { + const templateId = String(finding['template-id'] || finding.templateID || ''); + const matchedAt = String(finding['matched-at'] || finding.matchedAt || ''); + const info = (finding.info as any) || {}; + const severity = mapNucleiSeverity(String(finding.severity || info.severity || 'unknown')); + const name = String(info.name || finding.name || templateId); + const host = String(finding.host || ''); + + return { + scanner: 'nuclei', + finding_hash: generateFindingHash('nuclei', templateId, host, matchedAt), + severity, + asset_key: host || matchedAt, + rule_id: templateId, + title: name, + description: `Template: ${templateId} matched at ${matchedAt}`, + metadata: { + templateId, + matchedAt, + host, + tags: info.tags, + }, + }; +} + +/** + * Normalize Prowler finding + */ +function normalizeProwler(finding: Record): NormalizedFinding { + const checkId = String(finding.check_id || ''); + const status = String(finding.status || ''); + const severity = mapProwlerSeverity(String(finding.severity || 'medium')); + const resource = String(finding.resource || finding.resource_id || ''); + const checkTitle = String(finding.check_title || finding.title || checkId); + const statusExtended = String(finding.status_extended || finding.description || ''); + + return { + scanner: 'prowler', + finding_hash: generateFindingHash('prowler', checkId, resource), + severity, + asset_key: resource, + rule_id: checkId, + title: checkTitle, + description: statusExtended, + metadata: { + checkId, + status, + resource, + region: finding.region, + service: finding.service, + }, + }; +} + +// Severity mapping helpers +function mapDependencySeverity(severity: string): Severity { + const upper = severity.toUpperCase(); + if (upper === 'CRITICAL') return 'critical'; + if (upper === 'HIGH') return 'high'; + if (upper === 'MEDIUM' || upper === 'MODERATE') return 'medium'; + if (upper === 'LOW') return 'low'; + return 'info'; +} + +function mapTrivySeverity(severity: string): Severity { + const upper = severity.toUpperCase(); + if (upper === 'CRITICAL') return 'critical'; + if (upper === 'HIGH') return 'high'; + if (upper === 'MEDIUM') return 'medium'; + if (upper === 'LOW') return 'low'; + return 'info'; +} + +function mapNucleiSeverity(severity: string): Severity { + const lower = severity.toLowerCase(); + if (lower === 'critical') return 'critical'; + if (lower === 'high') return 'high'; + if (lower === 'medium') return 'medium'; + if (lower === 'low') return 'low'; + return 'info'; +} + +function mapProwlerSeverity(severity: string): Severity { + const lower = severity.toLowerCase(); + if (lower === 'critical') return 'critical'; + if (lower === 'high') return 'high'; + if (lower === 'medium') return 'medium'; + if (lower === 'low') return 'low'; + return 'info'; +} + +const definition = defineComponent({ + id: 'security.findings.normalize', + label: 'Normalize Security Findings', + category: 'security', + runner: { kind: 'inline' }, + inputs: inputSchema, + outputs: outputSchema, + parameters: parameterSchema, + docs: 'Convert scanner output to standardized format. Enables unified PR comments, Analytics Sink indexing, and filtering to changed files.', + ui: { + slug: 'security-findings-normalize', + version: '1.0.0', + type: 'output', + category: 'security', + description: 'Normalize findings from multiple scanners into a unified format.', + documentation: + 'Connect this component after any scanner (TruffleHog, OpenGrep, etc.) to convert findings to a standard format. The normalized output can be used with Analytics Sink for OpenSearch indexing or with findings formatters for PR comments.', + icon: 'Filter', + author: { + name: 'ShipSecAI', + type: 'shipsecai', + }, + isLatest: true, + deprecated: false, + example: 'Connect TruffleHog findings → Normalize → Analytics Sink for dashboards', + examples: [ + 'Normalize TruffleHog findings before posting PR comment', + 'Filter OpenGrep findings to only changed files in PR', + 'Combine multiple scanner outputs for unified reporting', + ], + }, + async execute({ inputs, params }, context) { + const { rawFindings, changedFiles } = inputSchema.parse(inputs); + const { scannerType, filterToChangedFiles } = parameterSchema.parse(params); + + context.logger.info(`${LOG_PREFIX} Normalizing ${rawFindings.length} findings`); + + if (rawFindings.length === 0) { + context.emitProgress({ + message: 'No findings to normalize', + level: 'info', + data: { stage: 'complete' }, + }); + + return outputSchema.parse({ + findings: [], + summary: { critical: 0, high: 0, medium: 0, low: 0, info: 0, total: 0 }, + findingCount: 0, + filteredCount: 0, + }); + } + + context.emitProgress({ + message: `Normalizing ${rawFindings.length} findings...`, + level: 'info', + data: { count: rawFindings.length, stage: 'start' }, + }); + + // Normalize each finding + const normalizedFindings: NormalizedFinding[] = []; + + for (const finding of rawFindings) { + const detectedType = scannerType === 'auto' ? detectScannerType(finding) : scannerType; + + let normalized: NormalizedFinding; + + switch (detectedType) { + case 'trufflehog': + normalized = normalizeTruffleHog(finding); + break; + case 'opengrep': + normalized = normalizeOpenGrep(finding); + break; + case 'dependency': + normalized = normalizeDependency(finding); + break; + case 'trivy': + normalized = normalizeTrivy(finding); + break; + case 'nuclei': + normalized = normalizeNuclei(finding); + break; + case 'prowler': + normalized = normalizeProwler(finding); + break; + default: + // Generic fallback - preserve as-is with basic normalization + normalized = { + scanner: 'unknown', + finding_hash: generateFindingHash('unknown', JSON.stringify(finding).slice(0, 100)), + severity: 'info', + rule_id: String(finding.id || finding.rule_id || 'unknown'), + title: String(finding.title || finding.message || 'Unknown finding'), + description: String(finding.description || ''), + file: stripWorkspacePrefix(String(finding.file || finding.path || '')), + line: Number(finding.line || 0), + metadata: finding, + }; + } + + normalizedFindings.push(normalized); + } + + // Filter to changed files if requested + let filteredFindings = normalizedFindings; + let filteredCount = 0; + + if (filterToChangedFiles && changedFiles && changedFiles.length > 0) { + // Build a set of normalized changed file paths (strip workspace prefixes) + const changedSet = new Set(); + for (const f of changedFiles) { + changedSet.add(stripWorkspacePrefix(f)); + } + + filteredFindings = normalizedFindings.filter((f) => { + if (!f.file) return true; // Keep findings without file info (e.g. dependency findings) + const normalizedPath = stripWorkspacePrefix(f.file); + return changedSet.has(normalizedPath); + }); + + filteredCount = normalizedFindings.length - filteredFindings.length; + + if (filteredCount > 0) { + context.logger.info( + `${LOG_PREFIX} Filtered ${filteredCount} findings (not in ${changedFiles.length} changed files)`, + ); + } + } + + // Build summary + const summary = { + critical: filteredFindings.filter((f) => f.severity === 'critical').length, + high: filteredFindings.filter((f) => f.severity === 'high').length, + medium: filteredFindings.filter((f) => f.severity === 'medium').length, + low: filteredFindings.filter((f) => f.severity === 'low').length, + info: filteredFindings.filter((f) => f.severity === 'info' || f.severity === 'none').length, + total: filteredFindings.length, + }; + + context.logger.info( + `${LOG_PREFIX} Normalized ${filteredFindings.length} findings (${summary.critical} critical, ${summary.high} high)`, + ); + + context.emitProgress({ + message: `✓ Normalized ${filteredFindings.length} findings`, + level: 'info', + data: { summary, filteredCount, stage: 'complete' }, + }); + + return outputSchema.parse({ + findings: filteredFindings, + summary, + findingCount: filteredFindings.length, + filteredCount, + }); + }, +}); + +componentRegistry.register(definition); diff --git a/worker/src/components/security/trufflehog.ts b/worker/src/components/security/trufflehog.ts index 72f4c19a6..e25d1bcf0 100644 --- a/worker/src/components/security/trufflehog.ts +++ b/worker/src/components/security/trufflehog.ts @@ -12,8 +12,11 @@ import { parameters, port, param, + generateFindingHash, + analyticsResultSchema, + type AnalyticsResult, } from '@shipsec/component-sdk'; -import { createIsolatedVolume } from '../../utils/isolated-volume'; +import { IsolatedContainerVolume } from '../../utils/isolated-volume'; const scanTypeSchema = z.enum(['git', 'github', 'gitlab', 's3', 'gcs', 'filesystem', 'docker']); @@ -66,7 +69,8 @@ const parameterSchema = parameters({ label: 'Only Verified', editor: 'boolean', description: 'Show only verified secrets (actively valid credentials).', - helpText: 'Disable to also show unverified potential secrets.', + helpText: + 'When enabled, passes --results=verified. When disabled, no --results flag is passed.', }), jsonOutput: param(z.boolean().default(true).describe('Output results in JSON format'), { label: 'JSON Output', @@ -186,6 +190,11 @@ const outputSchema = outputs({ label: 'Has Verified Secrets', description: 'True when any verified secrets are detected.', }), + results: port(z.array(analyticsResultSchema()), { + label: 'Results', + description: + 'Analytics-ready findings with scanner, finding_hash, and severity. Connect to Analytics Sink.', + }), }); // Helper function to build TruffleHog command arguments @@ -210,8 +219,6 @@ function buildTruffleHogCommand( // Add results filter if (input.onlyVerified) { args.push('--results=verified'); - } else { - args.push('--results=verified,unknown'); } // Add JSON output flag @@ -256,6 +263,7 @@ function parseRawOutput(rawOutput: string): Output { secretCount: 0, verifiedCount: 0, hasVerifiedSecrets: false, + results: [], }; } @@ -294,6 +302,7 @@ function parseRawOutput(rawOutput: string): Output { secretCount: secrets.length, verifiedCount, hasVerifiedSecrets: verifiedCount > 0, + results: [], // Populated in execute() with scanner metadata }; } @@ -303,7 +312,7 @@ const definition = defineComponent({ category: 'security', runner: { kind: 'docker', - image: 'ghcr.io/shipsecai/trufflehog:v3.93.1', + image: 'ghcr.io/shipsecai/trufflehog:latest', entrypoint: 'trufflehog', network: 'bridge', command: [], // Will be built dynamically in execute @@ -323,6 +332,11 @@ const definition = defineComponent({ outputs: outputSchema, parameters: parameterSchema, docs: 'Scan for secrets and credentials using TruffleHog. Supports Git repositories, GitHub, GitLab, filesystems, S3 buckets, Docker images, and more.', + toolProvider: { + kind: 'component', + name: 'secret_scan', + description: 'Secret and credential leakage scanner (TruffleHog).', + }, ui: { slug: 'trufflehog', version: '1.0.0', @@ -349,10 +363,6 @@ const definition = defineComponent({ 'Scan only changes in a Pull Request by setting branch to PR branch and sinceCommit to base branch.', 'Scan last 10 commits in CI/CD using sinceCommit=HEAD~10 to catch recent secrets.', ], - agentTool: { - enabled: true, - toolDescription: 'Secret and credential leakage scanner (TruffleHog).', - }, }, async execute({ inputs, params }, context) { const parsedParams = parameterSchema.parse(params); @@ -381,7 +391,7 @@ const definition = defineComponent({ }); // Handle filesystem scanning with isolated volumes - let volume: ReturnType | undefined; + let volume: IsolatedContainerVolume | undefined; let effectiveInput = runnerPayload; const baseRunner = definition.runner; @@ -404,7 +414,7 @@ const definition = defineComponent({ } const tenantId = (context as any).tenantId ?? 'default-tenant'; - volume = createIsolatedVolume(tenantId, context.runId); + volume = new IsolatedContainerVolume(tenantId, context.runId); // Initialize volume with files const volumeName = await volume.initialize(runnerPayload.filesystemContent); @@ -493,7 +503,23 @@ const definition = defineComponent({ }); } - return output; + // Build analytics-ready results with scanner metadata (follows core.analytics.result.v1 contract) + const results: AnalyticsResult[] = output.secrets.map((secret: Secret) => { + // Extract file path from source metadata for hashing + const filePath = + secret.SourceMetadata?.Data?.Git?.file ?? + secret.SourceMetadata?.Data?.Filesystem?.file ?? + ''; + return { + ...secret, + scanner: 'trufflehog', + severity: 'high' as const, // Secrets are always high severity + asset_key: runnerPayload.scanTarget, + finding_hash: generateFindingHash(secret.DetectorType, secret.Redacted, filePath), + }; + }); + + return { ...output, results }; } finally { // Always cleanup volume if it was created if (volume) { From 7abb35129b3a00c7744fa93ae7c9d6edbd10c9ec Mon Sep 17 00:00:00 2001 From: Aseem Shrey Date: Tue, 10 Feb 2026 20:53:42 -0500 Subject: [PATCH 017/690] feat(frontend): add GitHub integration UI with scan management and trigger UX --- frontend/src/App.tsx | 261 +- .../github/OperationsPreviewMock.tsx | 231 ++ frontend/src/components/layout/AppLayout.tsx | 33 +- frontend/src/components/layout/AppTopBar.tsx | 76 +- frontend/src/components/layout/Sidebar.tsx | 51 +- frontend/src/components/scans/FindingCard.tsx | 136 + .../src/components/scans/FindingsGroup.tsx | 59 + .../src/components/scans/ScanDetailPanel.tsx | 854 +++++ .../components/scans/SeverityDonutChart.tsx | 138 + .../scans/__tests__/scan-utils.test.ts | 91 + .../src/components/scans/findings-provider.ts | 66 + frontend/src/components/scans/scan-utils.ts | 327 ++ frontend/src/components/ui/sidebar.tsx | 5 +- .../src/components/workflow/ConfigPanel.tsx | 295 +- .../components/workflow/ParameterField.tsx | 267 +- .../components/workflow/node/WorkflowNode.tsx | 36 +- frontend/src/pages/CreateTriggerRulePage.tsx | 409 +++ frontend/src/pages/EditTriggerRulePage.tsx | 465 +++ frontend/src/pages/GitHubPage.tsx | 2937 +++++++++++++++++ frontend/src/pages/ScanResultPage.tsx | 739 +++++ frontend/src/pages/SchedulesPage.tsx | 131 +- frontend/src/pages/WorkflowList.tsx | 258 +- frontend/src/schemas/component.ts | 14 +- frontend/src/store/githubStore.ts | 668 ++++ frontend/src/utils/categoryColors.ts | 153 +- 25 files changed, 8234 insertions(+), 466 deletions(-) create mode 100644 frontend/src/components/github/OperationsPreviewMock.tsx create mode 100644 frontend/src/components/scans/FindingCard.tsx create mode 100644 frontend/src/components/scans/FindingsGroup.tsx create mode 100644 frontend/src/components/scans/ScanDetailPanel.tsx create mode 100644 frontend/src/components/scans/SeverityDonutChart.tsx create mode 100644 frontend/src/components/scans/__tests__/scan-utils.test.ts create mode 100644 frontend/src/components/scans/findings-provider.ts create mode 100644 frontend/src/components/scans/scan-utils.ts create mode 100644 frontend/src/pages/CreateTriggerRulePage.tsx create mode 100644 frontend/src/pages/EditTriggerRulePage.tsx create mode 100644 frontend/src/pages/GitHubPage.tsx create mode 100644 frontend/src/pages/ScanResultPage.tsx create mode 100644 frontend/src/store/githubStore.ts diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 48a7e2f9b..5d74956c9 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -1,12 +1,22 @@ -import { lazy, Suspense, useState, useEffect } from 'react'; import { BrowserRouter, Routes, Route } from 'react-router-dom'; -import { QueryClientProvider } from '@tanstack/react-query'; -const ReactQueryDevtools = lazy(() => - import('@tanstack/react-query-devtools').then((mod) => ({ - default: mod.ReactQueryDevtools, - })), -); -import { queryClient } from '@/lib/queryClient'; +import { WorkflowList } from '@/pages/WorkflowList'; +import { WorkflowBuilder } from '@/features/workflow-builder/WorkflowBuilder'; +import { SecretsManager } from '@/pages/SecretsManager'; +import { ApiKeysManager } from '@/pages/ApiKeysManager'; +import { IntegrationsManager } from '@/pages/IntegrationsManager'; +import { ArtifactLibrary } from '@/pages/ArtifactLibrary'; +import { McpLibraryPage } from '@/pages/McpLibraryPage'; +import { IntegrationCallback } from '@/pages/IntegrationCallback'; +import { NotFound } from '@/pages/NotFound'; +import { WebhooksPage } from '@/pages/WebhooksPage'; +import { WebhookEditorPage } from '@/pages/WebhookEditorPage'; +import { SchedulesPage } from '@/pages/SchedulesPage'; +import { ActionCenterPage } from '@/pages/ActionCenterPage'; +import { RunRedirect } from '@/pages/RunRedirect'; +import { GitHubPage } from '@/pages/GitHubPage'; +import { ScanResultPage } from '@/pages/ScanResultPage'; +import CreateTriggerRulePage from '@/pages/CreateTriggerRulePage'; +import EditTriggerRulePage from '@/pages/EditTriggerRulePage'; import { ToastProvider } from '@/components/ui/toast-provider'; import { AppLayout } from '@/components/layout/AppLayout'; import { AuthProvider } from '@/auth/auth-context'; @@ -14,91 +24,8 @@ import { useAuthStoreIntegration } from '@/auth/store-integration'; import { ProtectedRoute } from '@/components/auth/ProtectedRoute'; import { AnalyticsRouterListener } from '@/features/analytics/AnalyticsRouterListener'; import { PostHogClerkBridge } from '@/features/analytics/PostHogClerkBridge'; -import { useCommandPaletteKeyboard } from '@/features/command-palette/useCommandPaletteKeyboard'; -import { useCommandPaletteStore } from '@/store/commandPaletteStore'; -import { Skeleton } from '@/components/ui/skeleton'; - -// Lazy-loaded page components -const WorkflowList = lazy(() => - import('@/pages/WorkflowList').then((m) => ({ default: m.WorkflowList })), -); -const TemplateLibraryPage = lazy(() => - import('@/pages/TemplateLibraryPage').then((m) => ({ default: m.TemplateLibraryPage })), -); -const WorkflowBuilder = lazy(() => - import('@/features/workflow-builder/WorkflowBuilder').then((m) => ({ - default: m.WorkflowBuilder, - })), -); -const SecretsManager = lazy(() => - import('@/pages/SecretsManager').then((m) => ({ default: m.SecretsManager })), -); -const ApiKeysManager = lazy(() => - import('@/pages/ApiKeysManager').then((m) => ({ default: m.ApiKeysManager })), -); -const IntegrationsManager = lazy(() => - import('@/pages/IntegrationsManager').then((m) => ({ default: m.IntegrationsManager })), -); -const ArtifactLibrary = lazy(() => - import('@/pages/ArtifactLibrary').then((m) => ({ default: m.ArtifactLibrary })), -); -const McpLibraryPage = lazy(() => - import('@/pages/McpLibraryPage').then((m) => ({ default: m.McpLibraryPage })), -); -const IntegrationCallback = lazy(() => - import('@/pages/IntegrationCallback').then((m) => ({ default: m.IntegrationCallback })), -); -const NotFound = lazy(() => import('@/pages/NotFound').then((m) => ({ default: m.NotFound }))); -const WebhooksPage = lazy(() => - import('@/pages/WebhooksPage').then((m) => ({ default: m.WebhooksPage })), -); -const WebhookEditorPage = lazy(() => - import('@/pages/WebhookEditorPage').then((m) => ({ default: m.WebhookEditorPage })), -); -const SchedulesPage = lazy(() => - import('@/pages/SchedulesPage').then((m) => ({ default: m.SchedulesPage })), -); -const ActionCenterPage = lazy(() => - import('@/pages/ActionCenterPage').then((m) => ({ default: m.ActionCenterPage })), -); -const RunRedirect = lazy(() => - import('@/pages/RunRedirect').then((m) => ({ default: m.RunRedirect })), -); -const AnalyticsSettingsPage = lazy(() => - import('@/pages/AnalyticsSettingsPage').then((m) => ({ default: m.AnalyticsSettingsPage })), -); -const SettingsPage = lazy(() => - import('@/pages/SettingsPage').then((m) => ({ default: m.SettingsPage })), -); - -// Lazy-load CommandPalette — it pulls in the entire lucide-react barrel (~350KB) -const CommandPalette = lazy(() => - import('@/features/command-palette/CommandPalette').then((m) => ({ - default: m.CommandPalette, - })), -); - -function PageSkeleton() { - return ( -
-
- - -
-
- - - - -
- - - -
-
-
- ); -} +import { CommandPalette, useCommandPaletteKeyboard } from '@/features/command-palette'; +import { AnalyticsSettingsPage } from '@/pages/AnalyticsSettingsPage'; function AuthIntegration({ children }: { children: React.ReactNode }) { useAuthStoreIntegration(); @@ -107,23 +34,10 @@ function AuthIntegration({ children }: { children: React.ReactNode }) { function CommandPaletteProvider({ children }: { children: React.ReactNode }) { useCommandPaletteKeyboard(); - const isOpen = useCommandPaletteStore((state) => state.isOpen); - const [hasOpened, setHasOpened] = useState(false); - - useEffect(() => { - if (isOpen && !hasOpened) { - setHasOpened(true); - } - }, [isOpen, hasOpened]); - return ( <> {children} - {hasOpened && ( - - - - )} + ); } @@ -132,76 +46,69 @@ function App() { return ( - - - - - {/* Analytics wiring */} - - - - - }> - - } /> - } /> - - - - } - /> - - - - } - /> - - - - } - /> - } /> - } /> - } /> - } /> - } /> - } /> - } /> - } /> - } /> - } /> - } /> - } /> - } /> - } /> - } /> - } - /> - } /> - - - - - - - - {import.meta.env.DEV && import.meta.env.VITE_DISABLE_DEVTOOLS !== 'true' && ( - - - - )} - + + + + {/* Analytics wiring */} + + + + + + } /> + + + + } + /> + + + + } + /> + + + + } + /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } + /> + } /> + + + + + + ); diff --git a/frontend/src/components/github/OperationsPreviewMock.tsx b/frontend/src/components/github/OperationsPreviewMock.tsx new file mode 100644 index 000000000..99688c5be --- /dev/null +++ b/frontend/src/components/github/OperationsPreviewMock.tsx @@ -0,0 +1,231 @@ +import { Badge } from '@/components/ui/badge'; +import { Button } from '@/components/ui/button'; +import { Card, CardContent, CardDescription, CardHeader, CardTitle } from '@/components/ui/card'; +import { Activity, CalendarClock, Clock3, PauseCircle, PlayCircle } from 'lucide-react'; + +interface MockLiveScan { + id: string; + repository: string; + source: string; + workflow: string; + status: 'running' | 'queued'; + progress: number; + startedAt: string; +} + +interface MockScheduledScan { + id: string; + name: string; + repository: string; + branch: string; + cadence: string; + nextRun: string; + lastResult: 'success' | 'failure' | 'none'; +} + +const MOCK_LIVE_SCANS: MockLiveScan[] = [ + { + id: 'live-1', + repository: 'LuD1161/dvwa', + source: 'PR #87', + workflow: 'Unified AI Security Review', + status: 'running', + progress: 68, + startedAt: 'Started 2m ago', + }, + { + id: 'live-2', + repository: 'ShipSecAI/code-review-test-repo', + source: 'Manual', + workflow: 'Security Scan with PR Comments', + status: 'queued', + progress: 12, + startedAt: 'Queued 30s ago', + }, + { + id: 'live-3', + repository: 'LuD1161/git-test-repo', + source: 'Push main', + workflow: 'PR Security Check Run', + status: 'running', + progress: 44, + startedAt: 'Started 5m ago', + }, +]; + +const MOCK_SCHEDULED_SCANS: MockScheduledScan[] = [ + { + id: 'sched-1', + name: 'Nightly Dependency Audit', + repository: 'LuD1161/dvwa', + branch: 'main', + cadence: 'Daily at 02:00 UTC', + nextRun: 'Tonight, 02:00 UTC', + lastResult: 'success', + }, + { + id: 'sched-2', + name: 'Weekly Full Security Sweep', + repository: 'ShipSecAI/code-review-test-repo', + branch: 'main', + cadence: 'Every Monday, 06:00 UTC', + nextRun: 'Mon, Feb 16 at 06:00 UTC', + lastResult: 'failure', + }, + { + id: 'sched-3', + name: 'Hourly Secret Detection', + repository: 'LuD1161/git-test-repo', + branch: 'main', + cadence: 'Every hour', + nextRun: 'In 24 minutes', + lastResult: 'none', + }, +]; + +function ResultBadge({ result }: { result: MockScheduledScan['lastResult'] }) { + if (result === 'success') { + return ( + + Success + + ); + } + if (result === 'failure') { + return Failure; + } + return No runs yet; +} + +function ProgressBar({ value }: { value: number }) { + return ( +
+
+
+ ); +} + +export function OperationsPreviewMock() { + const runningCount = MOCK_LIVE_SCANS.filter((scan) => scan.status === 'running').length; + const queuedCount = MOCK_LIVE_SCANS.filter((scan) => scan.status === 'queued').length; + + return ( +
+ + + + + Operations Preview (Mock Data) + + + UX-only mock: demonstrates how users can monitor live and scheduled scans in one place. + + + +
+
+

Running now

+

{runningCount}

+
+
+

Queued

+

{queuedCount}

+
+
+

Scheduled active

+

{MOCK_SCHEDULED_SCANS.length}

+
+
+

Need attention

+

0

+
+
+ + + + + + + + Live Scans + + + Current executions across manual, push, and PR triggers. + + + + {MOCK_LIVE_SCANS.map((scan) => ( +
+
+

{scan.repository}

+ + {scan.status === 'running' ? 'Running' : 'Queued'} + + {scan.source} + {scan.startedAt} +
+

{scan.workflow}

+
+
+ +
+ + {scan.progress}% + +
+
+ + +
+
+ ))} + + + + + + + + Scheduled Scans + + + Recurring scans with next-run visibility and one-click run now. + + + + {MOCK_SCHEDULED_SCANS.map((schedule) => ( +
+
+

{schedule.name}

+ +
+

+ {schedule.repository} • {schedule.branch} +

+
+ + + {schedule.cadence} + + + + Next: {schedule.nextRun} + +
+
+ +
+
+ ))} + + +
+ ); +} diff --git a/frontend/src/components/layout/AppLayout.tsx b/frontend/src/components/layout/AppLayout.tsx index 25bd9360e..793e06d39 100644 --- a/frontend/src/components/layout/AppLayout.tsx +++ b/frontend/src/components/layout/AppLayout.tsx @@ -24,10 +24,9 @@ import { Zap, Webhook, ServerCog, - BarChart3, Settings, ChevronDown, - Package, + Github, } from 'lucide-react'; import React, { useState, useEffect, useCallback } from 'react'; import { useAuthStore } from '@/store/authStore'; @@ -39,8 +38,6 @@ import { useThemeStore } from '@/store/themeStore'; import { cn } from '@/lib/utils'; import { setMobilePlacementSidebarClose } from '@/components/layout/sidebar-state'; import { useCommandPaletteStore } from '@/store/commandPaletteStore'; -import { usePrefetchOnIdle } from '@/hooks/usePrefetchOnIdle'; -import { prefetchIdleRoutes, prefetchRoute } from '@/lib/prefetch-routes'; interface AppLayoutProps { children: React.ReactNode; @@ -67,12 +64,6 @@ function useIsMobile(breakpoint = 768) { } export function AppLayout({ children }: AppLayoutProps) { - usePrefetchOnIdle(); - - // Prefetch all route chunks during idle time - useEffect(() => { - prefetchIdleRoutes(); - }, []); const isMobile = useIsMobile(); const [sidebarOpen, setSidebarOpen] = useState(!isMobile); const [, setIsHovered] = useState(false); @@ -274,11 +265,6 @@ export function AppLayout({ children }: AppLayoutProps) { href: '/', icon: Workflow, }, - { - name: 'Template Library', - href: '/templates', - icon: Package, - }, { name: 'Schedules', href: '/schedules', @@ -308,16 +294,11 @@ export function AppLayout({ children }: AppLayoutProps) { href: '/artifacts', icon: Archive, }, - ...(env.VITE_OPENSEARCH_DASHBOARDS_URL - ? [ - { - name: 'Dashboards', - href: env.VITE_OPENSEARCH_DASHBOARDS_URL, - icon: BarChart3, - external: true, - }, - ] - : []), + { + name: 'GitHub', + href: '/github', + icon: Github, + }, ]; const settingsItems = [ @@ -493,7 +474,6 @@ export function AppLayout({ children }: AppLayoutProps) { prefetchRoute(item.href)} onClick={(e) => { // If modifier key is held (CMD+click, Ctrl+click), link opens in new tab // Don't update sidebar state in this case @@ -587,7 +567,6 @@ export function AppLayout({ children }: AppLayoutProps) { prefetchRoute(item.href)} onClick={(e) => { if (e.metaKey || e.ctrlKey || e.shiftKey) { return; diff --git a/frontend/src/components/layout/AppTopBar.tsx b/frontend/src/components/layout/AppTopBar.tsx index 2ded2e809..6800a813e 100644 --- a/frontend/src/components/layout/AppTopBar.tsx +++ b/frontend/src/components/layout/AppTopBar.tsx @@ -1,8 +1,10 @@ +import { useEffect, useState } from 'react'; import { useLocation, Link } from 'react-router-dom'; import { Button } from '@/components/ui/button'; -import { PanelLeftClose, PanelLeftOpen } from 'lucide-react'; +import { Loader2, PanelLeftClose, PanelLeftOpen } from 'lucide-react'; import { env } from '@/config/env'; import { cn } from '@/lib/utils'; +import { API_BASE_URL, getApiAuthHeaders } from '@/services/api'; interface AppTopBarProps { title?: string; @@ -24,6 +26,46 @@ export function AppTopBar({ isMobile = false, }: AppTopBarProps) { const location = useLocation(); + const [hasActiveScans, setHasActiveScans] = useState(false); + const [loadingActiveScans, setLoadingActiveScans] = useState(false); + + useEffect(() => { + let cancelled = false; + + const loadActiveScans = async () => { + try { + const headers = await getApiAuthHeaders(); + const response = await fetch( + `${API_BASE_URL}/api/v1/github/scans?status=running,pending&limit=1`, + { headers }, + ); + if (!response.ok) return; + const scans = (await response.json()) as unknown; + if (cancelled) return; + setHasActiveScans(Array.isArray(scans) && scans.length > 0); + } catch { + if (!cancelled) { + setHasActiveScans(false); + } + } finally { + if (!cancelled) { + setLoadingActiveScans(false); + } + } + }; + + setLoadingActiveScans(true); + loadActiveScans().catch(() => {}); + + const timer = window.setInterval(() => { + loadActiveScans().catch(() => {}); + }, 15000); + + return () => { + cancelled = true; + window.clearInterval(timer); + }; + }, []); // Determine page title and navigation based on current route const getPageInfo = () => { @@ -45,14 +87,6 @@ export function AppTopBar({ }; } - if (location.pathname === '/templates') { - return { - title: 'Template Library', - shortTitle: 'Templates', - subtitle: 'Browse and use pre-built workflow templates', - }; - } - if (location.pathname.startsWith('/schedules')) { return { title: 'Workflow Schedules', @@ -101,11 +135,11 @@ export function AppTopBar({ }; } - if (location.pathname === '/analytics-settings') { + if (location.pathname.startsWith('/github')) { return { - title: 'Analytics Settings', - shortTitle: 'Analytics', - subtitle: 'Configure data retention and storage settings', + title: 'GitHub Integration', + shortTitle: 'GitHub', + subtitle: 'Manage repositories, trigger rules, and scan results', }; } @@ -178,7 +212,21 @@ export function AppTopBar({
{/* Action buttons */} -
{actions}
+
+ {hasActiveScans && ( + + )} + {actions} +
); } diff --git a/frontend/src/components/layout/Sidebar.tsx b/frontend/src/components/layout/Sidebar.tsx index 92d2cdfe4..68b595f1d 100644 --- a/frontend/src/components/layout/Sidebar.tsx +++ b/frontend/src/components/layout/Sidebar.tsx @@ -1,6 +1,6 @@ import { useEffect, useState, useMemo, useRef } from 'react'; import * as LucideIcons from 'lucide-react'; -import { useComponents } from '@/hooks/queries/useComponentQueries'; +import { useComponentStore } from '@/store/componentStore'; import { Input } from '@/components/ui/input'; import { Button } from '@/components/ui/button'; import { @@ -13,8 +13,7 @@ import type { ComponentMetadata } from '@/schemas/component'; import { cn } from '@/lib/utils'; import { env } from '@/config/env'; import { Skeleton } from '@/components/ui/skeleton'; -import { COMPONENT_CATEGORY_ORDER, isComponentCategory } from '@shipsec/shared'; -import { getCategorySeparatorColor, getCategoryTextColorClass } from '@/utils/categoryColors'; +import { type ComponentCategory, getCategorySeparatorColor } from '@/utils/categoryColors'; import { useThemeStore } from '@/store/themeStore'; import { useWorkflowUiStore } from '@/store/workflowUiStore'; import { useWorkflowStore } from '@/store/workflowStore'; @@ -201,8 +200,7 @@ interface SidebarProps { } export function Sidebar({ canManageWorkflows = true }: SidebarProps) { - const { data: componentIndex, isLoading: loading, error: componentsError } = useComponents(); - const error = componentsError?.message ?? null; + const { getAllComponents, fetchComponents, loading, error } = useComponentStore(); const [searchQuery, setSearchQuery] = useState(''); const [viewMode, setViewMode] = useState('list'); const theme = useThemeStore((state) => state.theme); @@ -212,13 +210,24 @@ export function Sidebar({ canManageWorkflows = true }: SidebarProps) { const hasBranchInfo = Boolean(frontendBranch || backendBranch); // Get category accent color (for left border) - uses separator colors for brightness - const getCategoryAccentColor = (category: string): string => { - return getCategorySeparatorColor(category, isDarkMode); + const getCategoryAccentColor = (category: string): string | undefined => { + return getCategorySeparatorColor(category as ComponentCategory, isDarkMode); }; // Get category text color with good contrast in both light and dark modes const getCategoryTextColor = (category: string): string => { - return getCategoryTextColorClass(category); + const categoryColors: Record = { + input: 'text-blue-600 dark:text-blue-400', + transform: 'text-orange-600 dark:text-orange-400', + ai: 'text-purple-600 dark:text-purple-400', + mcp: 'text-teal-600 dark:text-teal-400', + security: 'text-red-600 dark:text-red-400', + scanners: 'text-rose-600 dark:text-rose-400', + it_ops: 'text-cyan-600 dark:text-cyan-400', + notification: 'text-pink-600 dark:text-pink-400', + output: 'text-green-600 dark:text-green-400', + }; + return categoryColors[category] || 'text-foreground'; }; // Custom scrollbar state @@ -229,8 +238,15 @@ export function Sidebar({ canManageWorkflows = true }: SidebarProps) { const scrollTimeoutRef = useRef(null); const isScrollingRef = useRef(false); + // Fetch components on mount + useEffect(() => { + fetchComponents().catch((error) => { + console.error('Failed to load components', error); + }); + }, [fetchComponents]); + const showDemoComponents = useWorkflowUiStore((state) => state.showDemoComponents); - const allComponents = componentIndex ? Object.values(componentIndex.byId) : []; + const allComponents = getAllComponents(); // Helper to identify demo components const isDemoComponent = (component: ComponentMetadata) => { @@ -285,6 +301,19 @@ export function Sidebar({ canManageWorkflows = true }: SidebarProps) { ); }, [filteredComponents]); + // Category display order + const categoryOrder = [ + 'input', + 'output', + 'notification', + 'security', + 'scanners', + 'mcp', + 'ai', + 'transform', + 'it_ops', + ] as const; + // Filter components based on search query const filteredComponentsByCategory = useMemo(() => { const filtered = searchQuery.trim() @@ -324,8 +353,8 @@ export function Sidebar({ canManageWorkflows = true }: SidebarProps) { // Sort categories by predefined order const sortedEntries = Object.entries(filtered).sort(([a], [b]) => { - const indexA = isComponentCategory(a) ? COMPONENT_CATEGORY_ORDER.indexOf(a) : -1; - const indexB = isComponentCategory(b) ? COMPONENT_CATEGORY_ORDER.indexOf(b) : -1; + const indexA = categoryOrder.indexOf(a as (typeof categoryOrder)[number]); + const indexB = categoryOrder.indexOf(b as (typeof categoryOrder)[number]); // If category not in order list, put it at the end if (indexA === -1 && indexB === -1) return 0; if (indexA === -1) return 1; diff --git a/frontend/src/components/scans/FindingCard.tsx b/frontend/src/components/scans/FindingCard.tsx new file mode 100644 index 000000000..e16f486f1 --- /dev/null +++ b/frontend/src/components/scans/FindingCard.tsx @@ -0,0 +1,136 @@ +import { useState } from 'react'; +import { Badge } from '@/components/ui/badge'; +import { Button } from '@/components/ui/button'; +import { Github, FileCode, ChevronDown, ChevronRight } from 'lucide-react'; +import type { ScanFinding, GitHubRepo } from '@/store/githubStore'; +import { SEVERITY_CONFIG } from './scan-utils'; +import { cn } from '@/lib/utils'; + +const WORKSPACE_PREFIXES = ['/workspace/repo/', '/scan/repo/', '/scan/', '/workspace/', '/repo/']; + +function stripWorkspacePrefix(filePath: string): string { + for (const prefix of WORKSPACE_PREFIXES) { + if (filePath.startsWith(prefix)) { + return filePath.slice(prefix.length); + } + } + if (filePath.startsWith('/')) { + return filePath.slice(1); + } + return filePath; +} + +interface FindingCardProps { + finding: ScanFinding; + repository: GitHubRepo | null; + commitSha: string | null; + highlighted?: boolean; + onClick?: () => void; +} + +export function FindingCard({ + finding, + repository, + commitSha, + highlighted, + onClick, +}: FindingCardProps) { + const [expanded, setExpanded] = useState(false); + const config = SEVERITY_CONFIG[finding.severity] ?? SEVERITY_CONFIG.info; + const cleanFile = finding.file ? stripWorkspacePrefix(finding.file) : ''; + const hasSnippet = Boolean(finding.snippet); + + const githubFileUrl = + repository?.htmlUrl && commitSha && cleanFile + ? `${repository.htmlUrl}/blob/${commitSha}/${cleanFile}${finding.line ? `#L${finding.line}` : ''}` + : null; + + const handleCardClick = () => { + onClick?.(); + if (hasSnippet) { + setExpanded(true); + } + }; + + return ( +
+
+
+
+ {config.label} + {finding.type} + {finding.ruleId && ( + ({finding.ruleId}) + )} + {finding.category && ( + + {finding.category} + + )} +
+ +
+ + {cleanFile} + {finding.line && ( + + :{finding.line} + {finding.endLine && finding.endLine !== finding.line && `-${finding.endLine}`} + + )} +
+ +

{finding.message}

+ + {hasSnippet && ( +
+ + {expanded && ( + 
+                  {finding.snippet}
+
+ )} +
+ )} +
+ + {githubFileUrl && ( + e.stopPropagation()} + > + + + )} +
+
+ ); +} diff --git a/frontend/src/components/scans/FindingsGroup.tsx b/frontend/src/components/scans/FindingsGroup.tsx new file mode 100644 index 000000000..5eb70943b --- /dev/null +++ b/frontend/src/components/scans/FindingsGroup.tsx @@ -0,0 +1,59 @@ +import { useState } from 'react'; +import { ChevronDown, ChevronRight } from 'lucide-react'; +import type { ScanFinding, GitHubRepo } from '@/store/githubStore'; +import { SEVERITY_CONFIG } from './scan-utils'; +import { FindingCard } from './FindingCard'; + +interface FindingsGroupProps { + severity: string; + findings: ScanFinding[]; + repository: GitHubRepo | null; + commitSha: string | null; + highlightedFindingId?: string; + defaultExpanded?: boolean; + onFindingClick?: (findingId: string) => void; +} + +export function FindingsGroup({ + severity, + findings, + repository, + commitSha, + highlightedFindingId, + defaultExpanded, + onFindingClick, +}: FindingsGroupProps) { + const [collapsed, setCollapsed] = useState( + defaultExpanded === undefined ? false : !defaultExpanded, + ); + const config = SEVERITY_CONFIG[severity] ?? SEVERITY_CONFIG.info; + + if (findings.length === 0) return null; + + return ( +
+ + {!collapsed && ( +
+ {findings.map((finding) => ( + onFindingClick(finding.id) : undefined} + /> + ))} +
+ )} +
+ ); +} diff --git a/frontend/src/components/scans/ScanDetailPanel.tsx b/frontend/src/components/scans/ScanDetailPanel.tsx new file mode 100644 index 000000000..e62d515d9 --- /dev/null +++ b/frontend/src/components/scans/ScanDetailPanel.tsx @@ -0,0 +1,854 @@ +import { useEffect, useState, useRef } from 'react'; +import { Link } from 'react-router-dom'; +import { Badge } from '@/components/ui/badge'; +import { Button } from '@/components/ui/button'; +import { Card, CardContent } from '@/components/ui/card'; +import { Skeleton } from '@/components/ui/skeleton'; +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuTrigger, +} from '@/components/ui/dropdown-menu'; +import { + Dialog, + DialogContent, + DialogDescription, + DialogHeader, + DialogTitle, +} from '@/components/ui/dialog'; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from '@/components/ui/select'; +import { + Github, + GitPullRequest, + GitBranch, + GitCommit, + PlayCircle, + CalendarClock, + ExternalLink, + X, + Timer, + Clock, + ShieldAlert, + ShieldX, + Shield, + ShieldCheck, + AlertCircle, + Loader2, + ChevronDown, + Download, + FileJson, + FileSpreadsheet, + RotateCcw, + ArrowLeft, + BarChart3, + Search, + Package, +} from 'lucide-react'; +import { + useGitHubStore, + type GitHubScanResult, + type GitHubRepo, + type TriggerRule, +} from '@/store/githubStore'; +import { useToast } from '@/components/ui/use-toast'; +import { api } from '@/services/api'; +import { + STATUS_COLORS, + STATUS_ICONS, + SEVERITY_CONFIG, + SEVERITY_ORDER, + formatRelativeTime, + formatDuration, + exportToJson, + exportToCsv, + groupFindingsBySeverity, + getScanFailureGuidance, + getScanSourceMeta, +} from './scan-utils'; +import { FindingsGroup } from './FindingsGroup'; +import { SeverityDonutChart } from './SeverityDonutChart'; + +// ── Types ───────────────────────────────────────────────────────── + +interface ScanDetailPanelProps { + scanId: string; + highlightedFindingId?: string; + onClose: () => void; + onFindingSelect: (findingId: string | null) => void; + isMobile?: boolean; +} + +// ── Rerun Scan Modal ────────────────────────────────────────────── + +interface WorkflowOption { + id: string; + name: string; + description?: string; +} + +interface WorkflowCompatibility { + requiresPrContext: boolean; + requiredRuntimeInputs: string[]; +} + +const PR_CONTEXT_RUNTIME_INPUT_IDS = new Set([ + 'prnumber', + 'pullrequestnumber', + 'pull_request_number', + 'pr_number', +]); + +function normalizeRuntimeInputId(value: unknown): string { + return typeof value === 'string' ? value.replace(/[\s-]/g, '').toLowerCase() : ''; +} + +function buildWorkflowCompatibility(runtimeInputs: unknown[]): WorkflowCompatibility { + const requiredRuntimeInputs = runtimeInputs + .map((input) => { + if (!input || typeof input !== 'object') return null; + const item = input as { id?: unknown; required?: unknown }; + const id = typeof item.id === 'string' ? item.id : null; + if (!id) return null; + const required = item.required !== false; + return required ? id : null; + }) + .filter((id): id is string => Boolean(id)); + + const requiresPrContext = requiredRuntimeInputs.some((id) => + PR_CONTEXT_RUNTIME_INPUT_IDS.has(normalizeRuntimeInputId(id)), + ); + + return { requiresPrContext, requiredRuntimeInputs }; +} + +const QUICK_SCANS = [ + { + id: 'secret-detection', + name: 'Secret Detection', + description: 'Scan for exposed secrets', + icon: ShieldAlert, + }, + { + id: 'sast-semgrep', + name: 'SAST (Semgrep)', + description: 'Static application security testing', + icon: Search, + }, + { + id: 'dependency-audit', + name: 'Dependency Audit', + description: 'Check for vulnerable dependencies', + icon: Package, + }, +]; + +function RerunScanModal({ + open, + onClose, + repository, + defaultBranch, + triggerRule, + onSuccess, +}: { + open: boolean; + onClose: () => void; + repository: GitHubRepo | null; + defaultBranch: string; + triggerRule: TriggerRule | null; + onSuccess: () => void; +}) { + const triggerScan = useGitHubStore((s) => s.triggerScan); + const { toast } = useToast(); + const [workflows, setWorkflows] = useState([]); + const [workflowCompatibility, setWorkflowCompatibility] = useState< + Record + >({}); + const [loadingWorkflows, setLoadingWorkflows] = useState(false); + const [selectedWorkflow, setSelectedWorkflow] = useState(''); + const [selectedBranch, setSelectedBranch] = useState(''); + const [triggering, setTriggering] = useState(null); + + useEffect(() => { + if (open) { + setLoadingWorkflows(true); + setWorkflowCompatibility({}); + setSelectedBranch(defaultBranch || repository?.defaultBranch || 'main'); + api.workflows + .list() + .then(async (data) => { + const opts = data.map((w) => ({ + id: w.id, + name: w.name, + description: w.description ?? undefined, + })); + setWorkflows(opts); + setSelectedWorkflow(triggerRule?.workflowId ?? ''); + + const compatibilityEntries = await Promise.all( + data.map(async (workflow) => { + try { + const response = await api.workflows.getRuntimeInputs(workflow.id); + const compatibility = buildWorkflowCompatibility(response.inputs ?? []); + return [workflow.id, compatibility] as const; + } catch { + return [ + workflow.id, + { requiresPrContext: false, requiredRuntimeInputs: [] } as WorkflowCompatibility, + ] as const; + } + }), + ); + setWorkflowCompatibility(Object.fromEntries(compatibilityEntries)); + }) + .catch(() => + toast({ + title: 'Error', + description: 'Failed to load workflows', + variant: 'destructive', + }), + ) + .finally(() => setLoadingWorkflows(false)); + } + }, [open, repository, defaultBranch, triggerRule, toast]); + + const selectedWorkflowCompatibility = selectedWorkflow + ? (workflowCompatibility[selectedWorkflow] ?? null) + : null; + const selectedWorkflowRequiresPrContext = + selectedWorkflowCompatibility?.requiresPrContext ?? false; + + const handleTrigger = async (workflowId: string) => { + if (!repository) return; + const compatibility = workflowCompatibility[workflowId]; + if (compatibility?.requiresPrContext) { + toast({ + title: 'PR context required', + description: + 'This workflow expects PR runtime inputs. Use a PR trigger rule instead of a manual rerun.', + variant: 'destructive', + }); + return; + } + setTriggering(workflowId); + try { + await triggerScan(repository.id, workflowId, selectedBranch || undefined); + toast({ + title: 'Scan started', + description: `Security scan started for ${repository.fullName}`, + }); + onClose(); + onSuccess(); + } catch (error) { + toast({ + title: 'Error', + description: error instanceof Error ? error.message : 'Failed to start scan', + variant: 'destructive', + }); + } finally { + setTriggering(null); + } + }; + + if (!repository) return null; + + return ( + !isOpen && onClose()}> + + + Re-run Security Scan + + Start a new scan for {repository.fullName} + + +
+
+ + +
+
+

Quick Scans

+
+ {QUICK_SCANS.map((scan) => { + const Icon = scan.icon; + return ( + + ); + })} +
+
+
+

Custom Workflows

+ {loadingWorkflows ? ( + + ) : workflows.length === 0 ? ( +

No custom workflows available

+ ) : ( +
+ + +
+ )} + {selectedWorkflowRequiresPrContext && ( +

+ This workflow requires PR context (for example `prNumber`) and cannot be rerun + manually from this dialog. +

+ )} +
+
+ + + ); +} + +// ── Summary Cards ───────────────────────────────────────────────── + +function SummaryCards({ scan }: { scan: GitHubScanResult }) { + const { summary } = scan; + const cards = [ + { + label: 'Critical', + count: summary?.critical ?? 0, + icon: , + colorClass: 'text-red-700 dark:text-red-400', + bgColorClass: 'bg-red-50 dark:bg-red-950/30', + }, + { + label: 'High', + count: summary?.high ?? 0, + icon: , + colorClass: 'text-orange-700 dark:text-orange-400', + bgColorClass: 'bg-orange-50 dark:bg-orange-950/30', + }, + { + label: 'Medium', + count: summary?.medium ?? 0, + icon: , + colorClass: 'text-yellow-700 dark:text-yellow-400', + bgColorClass: 'bg-yellow-50 dark:bg-yellow-950/30', + }, + { + label: 'Low', + count: summary?.low ?? 0, + icon: , + colorClass: 'text-blue-700 dark:text-blue-400', + bgColorClass: 'bg-blue-50 dark:bg-blue-950/30', + }, + ]; + + return ( +
+ {cards.map((card) => ( + + +
+
+

{card.label}

+

{card.count}

+
+
{card.icon}
+
+ + + ))} +
+ ); +} + +// ── Error Alert ─────────────────────────────────────────────────── + +function ErrorAlert({ message, suggestion }: { message: string; suggestion: string }) { + return ( +
+
+ +
+

Scan Failed

+

{message}

+

+ Suggested fix: {suggestion} +

+
+
+
+ ); +} + +// ── Loading Skeleton ────────────────────────────────────────────── + +function PanelSkeleton() { + return ( +
+
+
+ + + +
+
+ + + +
+
+
+ {[1, 2, 3, 4].map((i) => ( + + ))} +
+ +
+ ); +} + +// ── Main Panel Component ────────────────────────────────────────── + +export function ScanDetailPanel({ + scanId, + highlightedFindingId, + onClose, + onFindingSelect, + isMobile, +}: ScanDetailPanelProps) { + const getScanResult = useGitHubStore((s) => s.getScanResult); + const repositories = useGitHubStore((s) => s.repositories); + const triggerRules = useGitHubStore((s) => s.triggerRules); + const fetchScanResults = useGitHubStore((s) => s.fetchScanResults); + + const [scan, setScan] = useState(null); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(null); + const [showRerunModal, setShowRerunModal] = useState(false); + const scrollContainerRef = useRef(null); + + // Fetch scan detail + useEffect(() => { + setLoading(true); + setError(null); + getScanResult(scanId) + .then(setScan) + .catch((err) => setError(err instanceof Error ? err.message : 'Failed to load scan')) + .finally(() => setLoading(false)); + }, [scanId, getScanResult]); + + // Scroll to highlighted finding + useEffect(() => { + if (!highlightedFindingId || loading) return; + const timer = setTimeout(() => { + const el = document.getElementById(`finding-${highlightedFindingId}`); + if (el) { + el.scrollIntoView({ behavior: 'smooth', block: 'center' }); + } + }, 300); + return () => clearTimeout(timer); + }, [highlightedFindingId, loading]); + + // Keyboard: Escape to close + useEffect(() => { + const handler = (e: KeyboardEvent) => { + if (e.key === 'Escape') onClose(); + }; + window.addEventListener('keydown', handler); + return () => window.removeEventListener('keydown', handler); + }, [onClose]); + + if (loading) return ; + + if (error || !scan) { + return ( +
+
+ +
+

Scan Not Found

+

{error || 'Could not load scan details.'}

+ +
+ ); + } + + const repository = repositories.find((r) => r.id === scan.repositoryId) ?? null; + const repoFullName = repository?.fullName ?? 'Unknown Repository'; + const repoHtmlUrl = repository?.htmlUrl; + const duration = formatDuration(scan.startedAt, scan.completedAt); + const commitUrl = + repoHtmlUrl && scan.commitSha ? `${repoHtmlUrl}/commit/${scan.commitSha}` : null; + const prUrl = + repoHtmlUrl && scan.prNumber != null ? `${repoHtmlUrl}/pull/${scan.prNumber}` : null; + const branchUrl = + repoHtmlUrl && scan.branch ? `${repoHtmlUrl}/tree/${encodeURIComponent(scan.branch)}` : null; + const triggerRule = scan.triggerRuleId + ? (triggerRules.find((r) => r.id === scan.triggerRuleId) ?? null) + : null; + const findings = scan.findings ?? []; + const groupedFindings = groupFindingsBySeverity(findings); + const showErrorAlert = + (scan.status === 'failure' || scan.status === 'error') && scan.errorMessage; + const sourceMeta = getScanSourceMeta(scan); + const scheduleScanParams = new URLSearchParams({ tab: 'scans', source: 'schedule' }); + if (sourceMeta.scheduleId) { + scheduleScanParams.set('scheduleId', sourceMeta.scheduleId); + } + const scheduleScanLink = `/github?${scheduleScanParams.toString()}`; + const failureGuidance = getScanFailureGuidance(scan); + const exactCreatedAtLocal = new Date(scan.createdAt).toLocaleString(undefined, { + year: 'numeric', + month: 'short', + day: 'numeric', + hour: '2-digit', + minute: '2-digit', + second: '2-digit', + timeZoneName: 'short', + }); + + const donutData = [ + { + label: 'Critical', + value: scan.summary?.critical ?? 0, + color: SEVERITY_CONFIG.critical.color, + }, + { label: 'High', value: scan.summary?.high ?? 0, color: SEVERITY_CONFIG.high.color }, + { label: 'Medium', value: scan.summary?.medium ?? 0, color: SEVERITY_CONFIG.medium.color }, + { label: 'Low', value: scan.summary?.low ?? 0, color: SEVERITY_CONFIG.low.color }, + { label: 'Info', value: scan.summary?.info ?? 0, color: SEVERITY_CONFIG.info.color }, + ]; + + const handleExportJson = () => exportToJson(scan, repoFullName); + const handleExportCsv = () => exportToCsv(scan, repoFullName); + const handleRerunSuccess = () => { + fetchScanResults({ force: true }).catch(() => {}); + }; + + return ( +
+ {/* ── Header ─────────────────────────────────────────────── */} +
+ {/* Top row: repo/status + actions/close */} +
+
+ {isMobile && ( + + )} + + {repoHtmlUrl ? ( + + {repoFullName} + + + ) : ( + {repoFullName} + )} + + {STATUS_ICONS[scan.status]} + {scan.status} + +
+
+ {!isMobile && ( + <> + + + + + + + JSON + + + CSV + + + + + + + )} +
+
+ + {/* Meta row: source, commit, duration, time */} +
+
+ {sourceMeta.kind === 'pr' ? ( + <> + + {prUrl ? ( + + {sourceMeta.label + (sourceMeta.detail ? ` ${sourceMeta.detail}` : '')} + + ) : ( + + {sourceMeta.label + (sourceMeta.detail ? ` ${sourceMeta.detail}` : '')} + + )} + + ) : sourceMeta.kind === 'push' ? ( + <> + + {branchUrl ? ( + + {sourceMeta.detail ?? sourceMeta.label} + + ) : ( + {sourceMeta.detail ?? sourceMeta.label} + )} + + ) : sourceMeta.kind === 'schedule' ? ( + <> + + + {sourceMeta.detail ?? sourceMeta.label} + + + ) : ( + <> + + {sourceMeta.label} + + )} +
+ {scan.commitSha && ( +
+ + {commitUrl ? ( + + {scan.commitSha.slice(0, 7)} + + ) : ( + {scan.commitSha.slice(0, 7)} + )} +
+ )} + {duration && ( +
+ + {duration} +
+ )} +
+ + {formatRelativeTime(scan.createdAt)} +
+
+ + {isMobile && ( +
+ + + + + + + JSON + + + CSV + + + + +
+ )} +
+ +
+
+
+ + Overview + {findings.length > 0 && ( + + {findings.length} + + )} +
+
+ +
+
+ {showErrorAlert && ( + + )} + +
+ + + + + + +
+ + {findings.length === 0 ? ( +
+
+ +
+

No Findings

+

+ {scan.status === 'success' + ? 'No security issues were detected in this scan.' + : scan.status === 'running' || scan.status === 'pending' + ? 'Scan is still in progress. Findings will appear here when complete.' + : 'No findings available for this scan.'} +

+
+ ) : ( +
+ {SEVERITY_ORDER.map((severity) => { + const severityFindings = groupedFindings[severity]; + const hasHighlightedFinding = highlightedFindingId + ? severityFindings.some((f) => f.id === highlightedFindingId) + : false; + return ( + + ); + })} +
+ )} +
+
+
+ + {/* ── Rerun Modal ──────────────────────────────────────── */} + setShowRerunModal(false)} + repository={repository} + defaultBranch={scan.branch || ''} + triggerRule={triggerRule} + onSuccess={handleRerunSuccess} + /> +
+ ); +} diff --git a/frontend/src/components/scans/SeverityDonutChart.tsx b/frontend/src/components/scans/SeverityDonutChart.tsx new file mode 100644 index 000000000..e3c89efb3 --- /dev/null +++ b/frontend/src/components/scans/SeverityDonutChart.tsx @@ -0,0 +1,138 @@ +import { cn } from '@/lib/utils'; + +interface DonutSegment { + label: string; + value: number; + color: string; +} + +interface SeverityDonutChartProps { + data: DonutSegment[]; + size?: number; + strokeWidth?: number; + className?: string; + showLegend?: boolean; +} + +export function SeverityDonutChart({ + data, + size = 180, + strokeWidth = 28, + className, + showLegend = true, +}: SeverityDonutChartProps) { + const filtered = data.filter((d) => d.value > 0); + const total = filtered.reduce((sum, d) => sum + d.value, 0); + + if (total === 0) { + return ( +
+ + + + No findings + + +
+ ); + } + + const radius = (size - strokeWidth) / 2; + const circumference = 2 * Math.PI * radius; + const center = size / 2; + + // Build segments + let cumulativeOffset = 0; + const segments = filtered.map((segment) => { + const segmentLength = (segment.value / total) * circumference; + const gap = filtered.length > 1 ? 3 : 0; // gap between segments + const dashArray = `${Math.max(0, segmentLength - gap)} ${circumference}`; + const dashOffset = -cumulativeOffset; + cumulativeOffset += segmentLength; + + return { + ...segment, + dashArray, + dashOffset, + }; + }); + + return ( +
+
+ + {/* Background ring */} + + {/* Segments */} + {segments.map((segment, i) => ( + + ))} + + {/* Center text */} +
+ {total} + + {total === 1 ? 'finding' : 'findings'} + +
+
+ + {showLegend && ( +
+ {filtered.map((segment) => ( +
+
+ {segment.label} + {segment.value} +
+ ))} +
+ )} +
+ ); +} diff --git a/frontend/src/components/scans/__tests__/scan-utils.test.ts b/frontend/src/components/scans/__tests__/scan-utils.test.ts new file mode 100644 index 000000000..ea1e9b891 --- /dev/null +++ b/frontend/src/components/scans/__tests__/scan-utils.test.ts @@ -0,0 +1,91 @@ +import { describe, expect, it } from 'bun:test'; +import type { GitHubScanResult } from '@/store/githubStore'; +import { getScanFailureGuidance, getScanSourceMeta } from '../scan-utils'; + +function makeScan(overrides: Partial = {}): GitHubScanResult { + return { + id: 'scan-1', + repositoryId: 'repo-1', + workflowRunId: 'run-1', + sourceType: 'manual', + triggerType: null, + triggerSource: null, + triggerLabel: null, + scheduleId: null, + scheduleName: null, + prNumber: null, + branch: 'main', + commitSha: null, + status: 'failure', + summary: { critical: 0, high: 0, medium: 0, low: 0, info: 0 }, + findingsCount: 0, + findings: [], + checkRunId: null, + prCommentId: null, + prReviewId: null, + resultsUrl: null, + errorMessage: null, + triggerRuleId: null, + organizationId: 'org-1', + startedAt: null, + completedAt: null, + createdAt: '2026-02-10T00:00:00.000Z', + updatedAt: '2026-02-10T00:00:00.000Z', + workflowName: null, + workflowVersion: null, + ...overrides, + }; +} + +describe('scan-utils source metadata', () => { + it('maps schedule metadata when run trigger is schedule', () => { + const source = getScanSourceMeta( + makeScan({ + sourceType: 'manual', + triggerType: 'schedule', + triggerSource: 'sched-123', + triggerLabel: 'Nightly Dependency Audit', + }), + ); + + expect(source.kind).toBe('schedule'); + expect(source.scheduleId).toBe('sched-123'); + expect(source.scheduleName).toBe('Nightly Dependency Audit'); + expect(source.detail).toBe('Nightly Dependency Audit'); + }); + + it('maps PR metadata from sourceType/prNumber', () => { + const source = getScanSourceMeta( + makeScan({ + sourceType: 'pr', + prNumber: 42, + }), + ); + + expect(source.kind).toBe('pr'); + expect(source.label).toBe('PR'); + expect(source.detail).toBe('#42'); + }); +}); + +describe('scan-utils failure guidance', () => { + it('suggests PR-trigger remediation for missing PR context', () => { + const guidance = getScanFailureGuidance( + makeScan({ + errorMessage: 'Missing PR context: prNumber not provided for manual run.', + }), + ); + + expect(guidance.suggestion).toContain('PR trigger rule'); + }); + + it('suggests permission remediation for GitHub permission errors', () => { + const guidance = getScanFailureGuidance( + makeScan({ + errorMessage: 'GitHub comment API permission denied for installation.', + }), + ); + + expect(guidance.suggestion).toContain('permissions'); + }); +}); diff --git a/frontend/src/components/scans/findings-provider.ts b/frontend/src/components/scans/findings-provider.ts new file mode 100644 index 000000000..104ef5047 --- /dev/null +++ b/frontend/src/components/scans/findings-provider.ts @@ -0,0 +1,66 @@ +import type { ScanFinding, GitHubScanResult } from '@/store/githubStore'; + +export interface FindingsQuery { + scanId: string; + severity?: string[]; + search?: string; + limit?: number; + offset?: number; +} + +export interface FindingsResult { + findings: ScanFinding[]; + total: number; + hasMore: boolean; +} + +export interface FindingsProvider { + getFindings(query: FindingsQuery): Promise; + getFinding(scanId: string, findingId: string): Promise; +} + +/** + * Reads findings from the scan result's embedded findings array. + * This is the current implementation — findings are returned inline + * with the scan result from GET /api/v1/github/scans/:id. + * + * When OpenSearch is ready, swap to OpenSearchFindingsProvider + * which will query the findings index directly. + */ +export class EmbeddedFindingsProvider implements FindingsProvider { + constructor(private getScanResult: (id: string) => Promise) {} + + async getFindings(query: FindingsQuery): Promise { + const scan = await this.getScanResult(query.scanId); + let findings = scan.findings ?? []; + + if (query.severity?.length) { + findings = findings.filter((f) => query.severity!.includes(f.severity)); + } + + if (query.search) { + const term = query.search.toLowerCase(); + findings = findings.filter( + (f) => + f.message.toLowerCase().includes(term) || + f.file.toLowerCase().includes(term) || + f.type.toLowerCase().includes(term), + ); + } + + const total = findings.length; + const offset = query.offset ?? 0; + const limit = query.limit ?? findings.length; + + return { + findings: findings.slice(offset, offset + limit), + total, + hasMore: offset + limit < total, + }; + } + + async getFinding(scanId: string, findingId: string): Promise { + const scan = await this.getScanResult(scanId); + return scan.findings?.find((f) => f.id === findingId) ?? null; + } +} diff --git a/frontend/src/components/scans/scan-utils.ts b/frontend/src/components/scans/scan-utils.ts new file mode 100644 index 000000000..94c8955fe --- /dev/null +++ b/frontend/src/components/scans/scan-utils.ts @@ -0,0 +1,327 @@ +import { Badge } from '@/components/ui/badge'; +import type { GitHubScanResult, ScanFinding } from '@/store/githubStore'; +import { Check, X, Clock, AlertCircle, Loader2 } from 'lucide-react'; +import React from 'react'; + +// ── Status rendering ────────────────────────────────────────────── + +export const STATUS_COLORS: Record = { + pending: 'bg-yellow-500', + running: 'bg-blue-500', + success: 'bg-green-500', + failure: 'bg-red-500', + error: 'bg-red-700', +}; + +export const STATUS_ICONS: Record = { + pending: React.createElement(Clock, { className: 'h-4 w-4' }), + running: React.createElement(Loader2, { className: 'h-4 w-4 animate-spin' }), + success: React.createElement(Check, { className: 'h-4 w-4' }), + failure: React.createElement(X, { className: 'h-4 w-4' }), + error: React.createElement(AlertCircle, { className: 'h-4 w-4' }), +}; + +// ── Severity rendering ──────────────────────────────────────────── + +export const SEVERITY_CONFIG: Record< + string, + { label: string; badgeClass: string; bgClass: string; textClass: string; color: string } +> = { + critical: { + label: 'Critical', + badgeClass: 'bg-red-600 text-white', + bgClass: 'bg-red-50 dark:bg-red-950/30', + textClass: 'text-red-700 dark:text-red-400', + color: '#dc2626', + }, + high: { + label: 'High', + badgeClass: 'bg-orange-600 text-white', + bgClass: 'bg-orange-50 dark:bg-orange-950/30', + textClass: 'text-orange-700 dark:text-orange-400', + color: '#ea580c', + }, + medium: { + label: 'Medium', + badgeClass: 'bg-yellow-600 text-white', + bgClass: 'bg-yellow-50 dark:bg-yellow-950/30', + textClass: 'text-yellow-700 dark:text-yellow-400', + color: '#ca8a04', + }, + low: { + label: 'Low', + badgeClass: 'bg-blue-600 text-white', + bgClass: 'bg-blue-50 dark:bg-blue-950/30', + textClass: 'text-blue-700 dark:text-blue-400', + color: '#2563eb', + }, + info: { + label: 'Info', + badgeClass: 'bg-gray-600 text-white', + bgClass: 'bg-gray-50 dark:bg-gray-950/30', + textClass: 'text-gray-700 dark:text-gray-400', + color: '#4b5563', + }, +}; + +export const SEVERITY_ORDER = ['critical', 'high', 'medium', 'low', 'info'] as const; + +// ── Source mapping ─────────────────────────────────────────────── + +export type ScanSourceKind = 'pr' | 'push' | 'manual' | 'schedule'; + +export interface ScanSourceMeta { + kind: ScanSourceKind; + label: string; + detail: string | null; + scheduleId: string | null; + scheduleName: string | null; +} + +export interface ScanFailureGuidance { + reason: string; + suggestion: string; +} + +export function getScanSourceMeta(scan: GitHubScanResult): ScanSourceMeta { + const isSchedule = scan.sourceType === 'schedule' || scan.triggerType === 'schedule'; + if (isSchedule) { + return { + kind: 'schedule', + label: 'Scheduled', + detail: scan.scheduleName ?? scan.triggerLabel ?? null, + scheduleId: scan.scheduleId ?? scan.triggerSource ?? null, + scheduleName: scan.scheduleName ?? scan.triggerLabel ?? null, + }; + } + + if (scan.sourceType === 'pr') { + return { + kind: 'pr', + label: 'PR', + detail: scan.prNumber != null ? `#${scan.prNumber}` : null, + scheduleId: null, + scheduleName: null, + }; + } + + if (scan.sourceType === 'push') { + return { + kind: 'push', + label: 'Push', + detail: scan.branch ?? null, + scheduleId: null, + scheduleName: null, + }; + } + + return { + kind: 'manual', + label: 'Manual', + detail: null, + scheduleId: null, + scheduleName: null, + }; +} + +export function getScanFailureGuidance(scan: GitHubScanResult): ScanFailureGuidance { + const reason = scan.errorMessage?.trim() || 'Execution failed before completion.'; + const normalized = reason.toLowerCase(); + + if ( + normalized.includes('pr context') || + normalized.includes('prnumber') || + normalized.includes('pull request number') + ) { + return { + reason, + suggestion: 'Run from a PR trigger rule, or pick a workflow that does not require PR inputs.', + }; + } + + if ( + normalized.includes('permission denied') || + normalized.includes('insufficient permission') || + normalized.includes('write permission') + ) { + return { + reason, + suggestion: 'Update GitHub App permissions for pull requests/comments, then run again.', + }; + } + + if (normalized.includes('rate limit')) { + return { + reason, + suggestion: 'Wait for GitHub rate limits to reset, then retry the run.', + }; + } + + return { + reason, + suggestion: 'Open run details to inspect failed steps and retry after fixing the root cause.', + }; +} + +// ── Time formatting ─────────────────────────────────────────────── + +export function formatRelativeTime(dateString: string): string { + const date = new Date(dateString); + const now = new Date(); + const diffMs = now.getTime() - date.getTime(); + const diffMins = Math.floor(diffMs / 60000); + const diffHours = Math.floor(diffMs / 3600000); + const diffDays = Math.floor(diffMs / 86400000); + + if (diffMins < 1) return 'just now'; + if (diffMins < 60) return `${diffMins}m ago`; + if (diffHours < 24) return `${diffHours}h ago`; + if (diffDays < 7) return `${diffDays}d ago`; + return date.toLocaleDateString(); +} + +export function formatExactTime(dateString: string): string { + const date = new Date(dateString); + return date.toLocaleString(undefined, { + year: 'numeric', + month: 'short', + day: 'numeric', + hour: '2-digit', + minute: '2-digit', + second: '2-digit', + }); +} + +export function formatDuration( + startedAt: string | null, + completedAt: string | null, +): string | null { + if (!startedAt || !completedAt) return null; + + const start = new Date(startedAt); + const end = new Date(completedAt); + const diffMs = end.getTime() - start.getTime(); + + if (diffMs < 1000) return '<1s'; + if (diffMs < 60000) return `${Math.round(diffMs / 1000)}s`; + if (diffMs < 3600000) { + const mins = Math.floor(diffMs / 60000); + const secs = Math.round((diffMs % 60000) / 1000); + return secs > 0 ? `${mins}m ${secs}s` : `${mins}m`; + } + + const hours = Math.floor(diffMs / 3600000); + const mins = Math.round((diffMs % 3600000) / 60000); + return mins > 0 ? `${hours}h ${mins}m` : `${hours}h`; +} + +// ── Summary badges component ────────────────────────────────────── + +export function SummaryBadges({ summary }: { summary: GitHubScanResult['summary'] }) { + const items = [ + { label: 'C', count: summary.critical, color: 'bg-red-600' }, + { label: 'H', count: summary.high, color: 'bg-orange-500' }, + { label: 'M', count: summary.medium, color: 'bg-yellow-500' }, + { label: 'L', count: summary.low, color: 'bg-blue-500' }, + ].filter((item) => item.count > 0); + + if (items.length === 0) { + return React.createElement( + 'span', + { className: 'text-muted-foreground text-sm' }, + 'No findings', + ); + } + + return React.createElement( + 'div', + { className: 'flex gap-1' }, + items.map((item) => + React.createElement( + Badge, + { + key: item.label, + variant: 'outline' as const, + className: `${item.color} text-white border-none text-xs px-1.5`, + }, + `${item.label}:${item.count}`, + ), + ), + ); +} + +// ── Export helpers ───────────────────────────────────────────────── + +function formatDateForFilename(): string { + return new Date().toISOString().split('T')[0]; +} + +function sanitizeFilename(name: string): string { + return name + .replace(/[^a-zA-Z0-9_-]/g, '-') + .replace(/-+/g, '-') + .toLowerCase(); +} + +function downloadFile(content: string, filename: string, mimeType: string): void { + const blob = new Blob([content], { type: mimeType }); + const url = URL.createObjectURL(blob); + const link = document.createElement('a'); + link.href = url; + link.download = filename; + document.body.appendChild(link); + link.click(); + document.body.removeChild(link); + URL.revokeObjectURL(url); +} + +export function exportToJson(scan: GitHubScanResult, repoName: string): void { + const sanitizedRepoName = sanitizeFilename(repoName); + const date = formatDateForFilename(); + const filename = `scan-${sanitizedRepoName}-${date}.json`; + const content = JSON.stringify(scan, null, 2); + downloadFile(content, filename, 'application/json'); +} + +export function exportToCsv(scan: GitHubScanResult, repoName: string): void { + const sanitizedRepoName = sanitizeFilename(repoName); + const date = formatDateForFilename(); + const filename = `scan-${sanitizedRepoName}-${date}.csv`; + + const findings = scan.findings ?? []; + const headers = ['severity', 'type', 'file', 'line', 'message']; + + const rows = findings.map((finding) => { + const escapeField = (value: string | number | null | undefined): string => { + if (value === null || value === undefined) return ''; + const str = String(value); + if (str.includes(',') || str.includes('"') || str.includes('\n') || str.includes('\r')) { + return `"${str.replace(/"/g, '""')}"`; + } + return str; + }; + + return [ + escapeField(finding.severity), + escapeField(finding.type), + escapeField(finding.file), + escapeField(finding.line), + escapeField(finding.message), + ].join(','); + }); + + const content = [headers.join(','), ...rows].join('\n'); + downloadFile(content, filename, 'text/csv'); +} + +// ── Grouping utility ────────────────────────────────────────────── + +export function groupFindingsBySeverity(findings: ScanFinding[]): Record { + return SEVERITY_ORDER.reduce( + (acc, severity) => { + acc[severity] = findings.filter((f) => f.severity === severity); + return acc; + }, + {} as Record, + ); +} diff --git a/frontend/src/components/ui/sidebar.tsx b/frontend/src/components/ui/sidebar.tsx index c89e28316..f9a4c0264 100644 --- a/frontend/src/components/ui/sidebar.tsx +++ b/frontend/src/components/ui/sidebar.tsx @@ -21,10 +21,7 @@ const SidebarHeader = React.forwardRef (
), diff --git a/frontend/src/components/workflow/ConfigPanel.tsx b/frontend/src/components/workflow/ConfigPanel.tsx index 2e1749ff6..912324f02 100644 --- a/frontend/src/components/workflow/ConfigPanel.tsx +++ b/frontend/src/components/workflow/ConfigPanel.tsx @@ -27,9 +27,8 @@ import { SelectValue, } from '@/components/ui/select'; import { Badge } from '@/components/ui/badge'; -import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from '@/components/ui/tooltip'; import { cn } from '@/lib/utils'; -import { useComponents } from '@/hooks/queries/useComponentQueries'; +import { useComponentStore } from '@/store/componentStore'; import { ParameterFieldWrapper } from './ParameterField'; import { WebhookDetails } from './WebhookDetails'; import { SecretSelect } from '@/components/inputs/SecretSelect'; @@ -48,7 +47,8 @@ import { } from '@/utils/portUtils'; import { API_V1_URL, api } from '@/services/api'; import { useWorkflowStore } from '@/store/workflowStore'; -import { useApiKeys, useApiKeyUiStore } from '@/hooks/queries/useApiKeyQueries'; +import { useApiKeyStore } from '@/store/apiKeyStore'; +import { useGitHubStore } from '@/store/githubStore'; import type { WorkflowSchedule } from '@shipsec/shared'; import { useOptionalWorkflowSchedulesContext } from '@/features/workflow-builder/contexts/useWorkflowSchedulesContext'; import { formatScheduleTimestamp, scheduleStatusVariant } from './schedules-utils'; @@ -299,37 +299,94 @@ export function ConfigPanel({ onViewSchedules, }: ConfigPanelProps) { const isMobile = useIsMobile(); - const { data: componentIndex, isLoading: loading } = useComponents(); - const getComponent = (ref: string) => { - if (!componentIndex || !ref) return null; - if (componentIndex.byId[ref]) return componentIndex.byId[ref]; - const idFromSlug = componentIndex.slugIndex[ref]; - if (idFromSlug && componentIndex.byId[idFromSlug]) return componentIndex.byId[idFromSlug]; - return null; - }; + const { getComponent, loading } = useComponentStore(); const { getEdges, getNodes } = useReactFlow(); const fallbackWorkflowId = useWorkflowStore((state) => state.metadata.id); const workflowId = workflowIdProp ?? fallbackWorkflowId; const navigate = useNavigate(); const schedulesContext = useOptionalWorkflowSchedulesContext(); + // GitHub App installations for installation dropdown + const ghInstallations = useGitHubStore((s) => s.installations); + const ghFetchInstallations = useGitHubStore((s) => s.fetchInstallations); + // Get API key for curl command - const lastCreatedKey = useApiKeyUiStore((state) => state.lastCreatedKey); - // API keys are auto-fetched by useApiKeys() used elsewhere; just ensure they're loaded - useApiKeys(); + const lastCreatedKey = useApiKeyStore((state) => state.lastCreatedKey); + const fetchApiKeys = useApiKeyStore((state) => state.fetchApiKeys); + + // Fetch API keys on mount if not already loaded + useEffect(() => { + fetchApiKeys().catch(console.error); + }, [fetchApiKeys]); // Use lastCreatedKey (full key) if available, otherwise null (will show placeholder) const activeApiKey = lastCreatedKey || null; - // Fixed width on desktop, full width on mobile - const effectiveWidth = isMobile ? '100%' : PANEL_WIDTH; + const [panelWidth, setPanelWidth] = useState(initialWidth); + const isResizing = useRef(false); + const resizeRef = useRef(null); + const latestNodeConfigRef = useRef({ + params: {}, + inputOverrides: {}, + }); + + // Actual width to use - full width on mobile + const effectiveWidth = isMobile ? '100%' : panelWidth; + + const handleMouseDown = useCallback( + (e: React.MouseEvent) => { + // Disable resizing on mobile + if (isMobile) return; + e.preventDefault(); + isResizing.current = true; + document.body.style.cursor = 'col-resize'; + document.body.style.userSelect = 'none'; + }, + [isMobile], + ); + + useEffect(() => { + const handleMouseMove = (e: MouseEvent) => { + if (!isResizing.current) return; + const newWidth = window.innerWidth - e.clientX; + const clampedWidth = Math.min(MAX_PANEL_WIDTH, Math.max(MIN_PANEL_WIDTH, newWidth)); + setPanelWidth(clampedWidth); + onWidthChange?.(clampedWidth); + }; + + const handleMouseUp = () => { + isResizing.current = false; + document.body.style.cursor = ''; + document.body.style.userSelect = ''; + }; + + document.addEventListener('mousemove', handleMouseMove); + document.addEventListener('mouseup', handleMouseUp); + + return () => { + document.removeEventListener('mousemove', handleMouseMove); + document.removeEventListener('mouseup', handleMouseUp); + }; + }, [onWidthChange]); + + useEffect(() => { + if (!selectedNode) { + latestNodeConfigRef.current = { params: {}, inputOverrides: {} }; + return; + } + + const config = selectedNode.data.config ?? { params: {}, inputOverrides: {} }; + latestNodeConfigRef.current = { + ...config, + params: { ...(config.params ?? {}) }, + inputOverrides: { ...(config.inputOverrides ?? {}) }, + }; + }, [selectedNode]); const handleParamValueChange = (paramId: string, value: any) => { if (!selectedNode || !onUpdateNode) return; - - const nodeData: FrontendNodeData = selectedNode.data; - const config = nodeData.config || { params: {}, inputOverrides: {} }; + const config = latestNodeConfigRef.current ?? { params: {}, inputOverrides: {} }; let updatedParams = { ...(config.params ?? {}), @@ -342,19 +399,21 @@ export function ConfigPanel({ updatedParams[paramId] = value; } + const updatedConfig: FrontendNodeData['config'] = { + ...config, + params: updatedParams, + inputOverrides: { ...(config.inputOverrides ?? {}) }, + }; + latestNodeConfigRef.current = updatedConfig; + onUpdateNode(selectedNode.id, { - config: { - ...config, - params: updatedParams, - }, + config: updatedConfig, }); }; const handleInputOverrideChange = (inputId: string, value: any) => { if (!selectedNode || !onUpdateNode) return; - - const nodeData: FrontendNodeData = selectedNode.data; - const config = nodeData.config || { params: {}, inputOverrides: {} }; + const config = latestNodeConfigRef.current ?? { params: {}, inputOverrides: {} }; let updatedOverrides = { ...(config.inputOverrides ?? {}), @@ -367,11 +426,15 @@ export function ConfigPanel({ updatedOverrides[inputId] = value; } + const updatedConfig: FrontendNodeData['config'] = { + ...config, + params: { ...(config.params ?? {}) }, + inputOverrides: updatedOverrides, + }; + latestNodeConfigRef.current = updatedConfig; + onUpdateNode(selectedNode.id, { - config: { - ...config, - inputOverrides: updatedOverrides, - }, + config: updatedConfig, }); }; @@ -384,7 +447,7 @@ export function ConfigPanel({ const isToolMode = Boolean( (nodeData.config as any)?.isToolMode || (nodeData.config as any)?.mode === 'tool', ); - const component = componentRef ? getComponent(componentRef) : null; + const component = getComponent(componentRef); if (!component) { if (loading) { @@ -510,6 +573,15 @@ export function ConfigPanel({ }; }, [component?.id, JSON.stringify(manualParameters), JSON.stringify(inputOverrides)]); // Deep compare parameters and overrides + // Lazy-fetch GitHub installations when a GitHub component is selected + const isGitHubAppComponent = component?.id?.startsWith('github.') ?? false; + const ghFetchedRef = useRef(false); + useEffect(() => { + if (!isGitHubAppComponent || ghFetchedRef.current) return; + ghFetchedRef.current = true; + ghFetchInstallations().catch(() => {}); + }, [isGitHubAppComponent, ghFetchInstallations]); + const componentInputs = dynamicInputs ?? component.inputs ?? []; const componentOutputs = dynamicOutputs ?? component.outputs ?? []; const componentParameters = component.parameters ?? []; @@ -1090,29 +1162,135 @@ export function ConfigPanel({ } }} disabled={manualLocked} - placeholder="{{run_id}}-{{timestamp}}" - /> - ) : ( - + + + + + True + False + + + {!manualLocked && typeof manualValue === 'boolean' && ( + + )} +
+ ) : isListOfTextInput ? ( + handleInputOverrideChange(input.id, value)} + /> + ) : isGitHubAppComponent && input.id === 'installationId' ? ( +
+ + {ghInstallations.length > 0 && + manualValue !== undefined && + manualValue !== null && ( +
+ {(() => { + const selected = ghInstallations.find( + (i) => i.installationId === Number(manualValue), + ); + if (!selected) return null; + return ( + <> + {selected.accountAvatarUrl && ( + {selected.accountLogin} + )} + + {selected.accountLogin} + + + {selected.accountType} + + + ); + })()} +
+ )} +

+ Note: Installation ID is automatically injected at runtime for + triggered scans. Only set this for manual testing. +

+
+ ) : component?.id === 'core.artifact.writer' && + input.id === 'artifactName' ? ( + { + if (!value || value === '') { + handleInputOverrideChange(input.id, undefined); + } else { + handleInputOverrideChange(input.id, value); + } + }} + disabled={manualLocked} + placeholder="{{run_id}}-{{timestamp}}" + /> + ) : ( + { + const nextValue = e.target.value; + if (nextValue === '') { + handleInputOverrideChange(input.id, undefined); + return; + } + if (isNumberInput) { + const parsed = Number(nextValue); + if (Number.isNaN(parsed)) { + return; + } + handleInputOverrideChange(input.id, parsed); + } else { + handleInputOverrideChange(input.id, nextValue); + } + }} placeholder={manualPlaceholder} className="text-sm" disabled={manualLocked} @@ -1330,22 +1508,13 @@ export function ConfigPanel({ key={schedule.id} className="rounded-lg border bg-background px-3 py-2 space-y-2" > -
-
-
- - - - - {schedule.name} - - - {schedule.name} - - +
+
+
+ {schedule.name} {schedule.status} @@ -1354,7 +1523,7 @@ export function ConfigPanel({ Next: {formatScheduleTimestamp(schedule.nextRunAt)}
-
+
); diff --git a/frontend/src/components/workflow/node/WorkflowNode.tsx b/frontend/src/components/workflow/node/WorkflowNode.tsx index fffdec5d8..51c283ec4 100644 --- a/frontend/src/components/workflow/node/WorkflowNode.tsx +++ b/frontend/src/components/workflow/node/WorkflowNode.tsx @@ -24,7 +24,7 @@ import { cn } from '@/lib/utils'; import { MarkdownView } from '@/components/ui/markdown'; import { Badge } from '@/components/ui/badge'; import { Button } from '@/components/ui/button'; -import { useComponents } from '@/hooks/queries/useComponentQueries'; +import { useComponentStore } from '@/store/componentStore'; import { useExecutionStore } from '@/store/executionStore'; import { useExecutionTimelineStore, type NodeVisualState } from '@/store/executionTimelineStore'; import { useWorkflowStore } from '@/store/workflowStore'; @@ -34,6 +34,7 @@ import type { InputPort } from '@/schemas/component'; import { useWorkflowUiStore } from '@/store/workflowUiStore'; import { useThemeStore } from '@/store/themeStore'; import { + type ComponentCategory, getCategorySeparatorColor, getCategoryHeaderBackgroundColor, } from '@/utils/categoryColors'; @@ -43,11 +44,11 @@ import { isCredentialInput, } from '@/utils/portUtils'; import { WebhookDetails } from '../WebhookDetails'; -import { useApiKeyUiStore } from '@/hooks/queries/useApiKeyQueries'; +import { useApiKeyStore } from '@/store/apiKeyStore'; import { API_V1_URL } from '@/services/api'; import { useNavigate, useParams } from 'react-router-dom'; import { useEntryPointActions } from '../entry-point-context'; -import { useSecrets } from '@/hooks/queries/useSecretQueries'; +import { useSecretStore } from '@/store/secretStore'; import { getSecretLabel } from '@/api/secrets'; import { useIsMobile } from '@/hooks/useIsMobile'; @@ -79,15 +80,8 @@ const TOOL_MODE_ONLY_COMPONENTS = new Set([ */ export const WorkflowNode = ({ data, selected, id }: NodeProps) => { // Store hooks - const { data: componentIndex, isLoading: loading } = useComponents(); - const getComponent = (ref: string) => { - if (!componentIndex || !ref) return null; - if (componentIndex.byId[ref]) return componentIndex.byId[ref]; - const idFromSlug = componentIndex.slugIndex[ref]; - if (idFromSlug && componentIndex.byId[idFromSlug]) return componentIndex.byId[idFromSlug]; - return null; - }; - const { data: secrets = [] } = useSecrets(); + const { getComponent, loading } = useComponentStore(); + const secrets = useSecretStore((state) => state.secrets); const { getNodes, getEdges, setNodes, deleteElements } = useReactFlow(); const updateNodeInternals = useUpdateNodeInternals(); const { nodeStates, selectedRunId, selectNode, isPlaying, playbackMode, isLiveFollowing } = @@ -98,7 +92,7 @@ export const WorkflowNode = ({ data, selected, id }: NodeProps) => { const prefetchTerminal = useExecutionStore((state) => state.prefetchTerminal); const terminalSession = useExecutionStore((state) => state.getTerminalSession(id, 'pty')); const theme = useThemeStore((state) => state.theme); - const lastCreatedKey = useApiKeyUiStore((state) => state.lastCreatedKey); + const { lastCreatedKey } = useApiKeyStore(); // @ts-expect-error - FIXME: Check actual store structure const workflowIdFromStore = useWorkflowStore((state) => state.workflow?.id); const { @@ -154,11 +148,12 @@ export const WorkflowNode = ({ data, selected, id }: NodeProps) => { // Cast to access extended frontend fields const nodeData = data as FrontendNodeData; const componentRef: string | undefined = nodeData.componentId ?? nodeData.componentSlug; - const component = componentRef ? getComponent(componentRef) : null; + const component = getComponent(componentRef); const isTextBlock = component?.id === 'core.ui.text'; const isEntryPoint = component?.id === 'core.workflow.entrypoint'; const isDarkMode = theme === 'dark'; - const componentCategory = component?.category ?? 'input'; + const componentCategory: ComponentCategory = + (component?.category as ComponentCategory) || (isEntryPoint ? 'input' : 'input'); const isToolModeOnly = component?.id ? TOOL_MODE_ONLY_COMPONENTS.has(component.id) : false; const showMcpBadge = componentCategory === 'mcp' || isToolModeOnly; const isToolMode = Boolean( @@ -1033,6 +1028,8 @@ export const WorkflowNode = ({ data, selected, id }: NodeProps) => { input.editor === 'secret' || (input.connectionType?.kind === 'primitive' && input.connectionType.name === 'secret'); + const isGitHubCloneRepositoryInput = + component?.id === 'github.repo.clone' && input.id === 'repository'; if (isSecretInput && manualDisplayVal) { const secret = secrets.find( @@ -1042,7 +1039,9 @@ export const WorkflowNode = ({ data, selected, id }: NodeProps) => { } const previewText = - manualDisplay.length > 24 ? `${manualDisplay.slice(0, 24)}…` : manualDisplay; + isGitHubCloneRepositoryInput || manualDisplay.length <= 24 + ? manualDisplay + : `${manualDisplay.slice(0, 24)}…`; const handleClassName = cn( '!w-[10px] !h-[10px] !border-2 !rounded-full', input.required @@ -1070,7 +1069,10 @@ export const WorkflowNode = ({ data, selected, id }: NodeProps) => {
s.createTriggerRule); + + // Form state + const [name, setName] = useState(''); + const [description, setDescription] = useState(''); + const [repositoryPattern, setRepositoryPattern] = useState(''); + const [event, setEvent] = useState('pull_request'); + const [actions, setActions] = useState(['opened', 'synchronize', 'reopened']); + const [branches, setBranches] = useState([]); + const [branchInput, setBranchInput] = useState(''); + const [workflowId, setWorkflowId] = useState(''); + const [postPrComment, setPostPrComment] = useState(true); + const [postPrReview, setPostPrReview] = useState(false); + const [createCheckRun, setCreateCheckRun] = useState(true); + const [failOn, setFailOn] = useState('high'); + const [enabled, setEnabled] = useState(true); + + // Workflows + const [workflows, setWorkflows] = useState([]); + const [workflowsLoading, setWorkflowsLoading] = useState(true); + + // Submit state + const [submitting, setSubmitting] = useState(false); + + useEffect(() => { + async function fetchWorkflows() { + try { + const list = await api.workflows.list(); + setWorkflows(list.map((w) => ({ id: w.id, name: w.name, description: w.description }))); + } catch { + toast({ title: 'Error', description: 'Failed to fetch workflows', variant: 'destructive' }); + } finally { + setWorkflowsLoading(false); + } + } + fetchWorkflows(); + }, [toast]); + + const handleActionToggle = (action: string) => { + setActions((prev) => + prev.includes(action) ? prev.filter((a) => a !== action) : [...prev, action], + ); + }; + + const handleAddBranch = () => { + const trimmed = branchInput.trim(); + if (trimmed && !branches.includes(trimmed)) { + setBranches((prev) => [...prev, trimmed]); + setBranchInput(''); + } + }; + + const handleRemoveBranch = (branch: string) => { + setBranches((prev) => prev.filter((b) => b !== branch)); + }; + + const handleSubmit = async (e: React.FormEvent) => { + e.preventDefault(); + + // Validation + if (!name.trim()) { + toast({ title: 'Validation Error', description: 'Name is required', variant: 'destructive' }); + return; + } + if (!repositoryPattern.trim()) { + toast({ + title: 'Validation Error', + description: 'Repository pattern is required', + variant: 'destructive', + }); + return; + } + if (!workflowId) { + toast({ + title: 'Validation Error', + description: 'Workflow is required', + variant: 'destructive', + }); + return; + } + + setSubmitting(true); + try { + const input: CreateTriggerRuleInput = { + name: name.trim(), + description: description.trim() || undefined, + repositoryPattern: repositoryPattern.trim(), + event, + actions: event === 'pull_request' ? actions : undefined, + branches: + event === 'repository_added' ? undefined : branches.length > 0 ? branches : undefined, + workflowId, + postPrComment, + postPrReview: event === 'pull_request' ? postPrReview : false, + createCheckRun, + failOn: createCheckRun ? failOn : undefined, + enabled, + }; + + await createTriggerRule(input); + toast({ title: 'Success', description: 'Trigger rule created' }); + navigate('/github?tab=triggers'); + } catch (error) { + const message = error instanceof Error ? error.message : 'Failed to create trigger rule'; + toast({ title: 'Error', description: message, variant: 'destructive' }); + } finally { + setSubmitting(false); + } + }; + + return ( +
+
+ +
+ + + + Create Trigger Rule + + Configure a rule to automatically trigger security scans when events occur. + + + +
+
+
+
+

Rule Scope

+

+ Define when this trigger should run. +

+
+ +
+ + setName(e.target.value)} + placeholder="e.g., Scan all PRs" + /> +
+ +
+ +