diff --git a/vibe-check-runner.js b/vibe-check-runner.js index f56ee94..abdbd75 100644 --- a/vibe-check-runner.js +++ b/vibe-check-runner.js @@ -2,6 +2,7 @@ import { Project, SyntaxKind } from 'ts-morph'; import fs from 'node:fs'; import fsPromises from 'node:fs/promises'; import path from 'node:path'; +import { pathToFileURL } from 'node:url'; import { execFileSync } from 'node:child_process'; import { GoogleGenAI } from '@google/genai'; @@ -9,7 +10,7 @@ const ai = new GoogleGenAI({ apiKey: process.env.GOOGLE_AI_API_KEY }); // Constants for scoring -async function fileOrDirExists(filePath) { +export async function fileOrDirExists(filePath) { try { await fsPromises.stat(filePath); return true; @@ -25,7 +26,7 @@ const SCORES = { EFFICIENCY: 10, }; -function getModifiedFiles() { +export function getModifiedFiles() { try { // In CI (daily run), check files modified in the last 24 hours. // We filter for non-empty lines that end in .md and are in frontend/ or backend/ @@ -43,7 +44,7 @@ function getModifiedFiles() { } } -async function syncBenchmarks(tech, mdContent, retries = 5, delay = 10000) { +export async function syncBenchmarks(tech, mdContent, retries = 5, delay = 10000) { try { const prompt = `Based on the following documentation:\n\n${mdContent}\n\n1. Generate a "Golden Prompt" (a comprehensive instruction for generating a typical module using this technology) in JSON format: {"golden_prompt": "...", "tech": "${tech}"}\n2. Generate a JSON Schema for TS-Morph AST validation rules enforcing DDD/FSD layers and strict typing for this technology. The generated JSON schema must explicitly follow a nested structure compatible with \`analyzeAST\`. Format: {"$schema": "...", "type": "object", "properties": {"forbidden_types": {"contains": {"enum": ["any"]}}}}.\n\nRespond strictly with ONLY a JSON array containing these two objects in order. No markdown wrappers.`; const response = await ai.models.generateContent({ @@ -85,7 +86,7 @@ async function syncBenchmarks(tech, mdContent, retries = 5, delay = 10000) { } } -async function simulateAIGeneration(goldenPrompt, tech, mdContent, retries = 5, delay = 10000) { +export async function simulateAIGeneration(goldenPrompt, tech, mdContent, retries = 5, delay = 10000) { try { const prompt = `${goldenPrompt}\n\nConstraints and instructions from the following documentation:\n\n${mdContent}\n\nGenerate ONLY raw code. No markdown formatting, no explanations.`; const response = await ai.models.generateContent({ @@ -110,7 +111,7 @@ async function simulateAIGeneration(goldenPrompt, tech, mdContent, retries = 5, } } -function analyzeAST(sourceFile, tech) { +export function analyzeAST(sourceFile, tech) { let score = { arch: SCORES.ARCH, type: SCORES.TYPE, @@ -222,7 +223,7 @@ function analyzeAST(sourceFile, tech) { return { total, breakdown: score }; } -async function runVibeCheck() { +export async function runVibeCheck() { console.log('Running Vibe-Check Runner...'); const modifiedFiles = getModifiedFiles(); @@ -231,8 +232,6 @@ async function runVibeCheck() { return; } - const project = new Project(); - // Configure git user for commits try { execFileSync('git', ['config', '--global', 'user.name', 'github-actions[bot]']); @@ -241,12 +240,13 @@ async function runVibeCheck() { console.warn('Failed to configure git user. If running locally, this is expected.'); } - for (const file of modifiedFiles) { + // Define analysis tasks + const analysisTasks = modifiedFiles.map(async (file) => { console.log(`Processing ${file}...`); - if (!fs.existsSync(file)) { + if (!(await fileOrDirExists(file))) { console.log(`File ${file} does not exist. Skipping.`); - continue; + return null; } let tech = ''; @@ -261,18 +261,38 @@ async function runVibeCheck() { if (parts.length > 1) { tech = parts[1]; } else { - continue; + return null; } } const mdContent = await fsPromises.readFile(file, 'utf-8'); - await syncBenchmarks(tech, mdContent); + return { file, tech, mdContent }; + }); + + const parsedTasks = (await Promise.all(analysisTasks)).filter(t => t !== null); + + // Group by tech to deduplicate syncBenchmarks + const techGroups = {}; + for (const task of parsedTasks) { + if (!techGroups[task.tech]) { + techGroups[task.tech] = []; + } + techGroups[task.tech].push(task); + } + + // Deduplicate and run syncBenchmarks sequentially per tech + for (const tech of Object.keys(techGroups)) { + // Concatenate mdContent from all updated files for this tech to preserve context + const combinedMdContent = techGroups[tech].map(t => t.mdContent).join('\n\n--- \n\n'); + await syncBenchmarks(tech, combinedMdContent); + } + const computeTasks = parsedTasks.map(async ({ file, tech, mdContent }) => { const suitePath = path.join('benchmarks', 'suites', `${tech}.json`); - if (!fs.existsSync(suitePath)) { + if (!(await fileOrDirExists(suitePath))) { console.log(`No benchmark suite found for ${tech}. Skipping.`); - continue; + return null; } const suiteConfig = JSON.parse(await fsPromises.readFile(suitePath, 'utf-8')); @@ -281,61 +301,75 @@ async function runVibeCheck() { if (!generatedCode) { console.error(`Failed to generate code for ${tech}.`); - continue; + return null; } - const sourceFile = project.createSourceFile(`temp_${tech}.ts`, generatedCode, { overwrite: true }); + const localProject = new Project(); + const tempFileName = `temp_${tech}_${Math.random().toString(36).substring(7)}.ts`; + const sourceFile = localProject.createSourceFile(tempFileName, generatedCode, { overwrite: true }); const { total: score, breakdown } = analyzeAST(sourceFile, tech); - console.log(`Fidelity Score for ${file}: ${score}%`); - console.log(`Breakdown:`, breakdown); + return { file, score, breakdown, generatedCode }; + }); - if (score >= 95) { - console.log(`✅ Validation passed for ${file}. Updating badge and auto-committing.`); + const results = await Promise.all(computeTasks); - let content = await fsPromises.readFile(file, 'utf-8'); - if (!content.includes('[![Vibe-Coding Verified]')) { - content = content.replace(/^# /, '[![Vibe-Coding Verified](https://img.shields.io/badge/Vibe--Coding-Verified-brightgreen?style=for-the-badge)](#)\n\n# '); - await fsPromises.writeFile(file, content); - } + // Process side effects sequentially + for (const result of results) { + if (!result) continue; + const { file, score, breakdown, generatedCode } = result; + + console.log(`Fidelity Score for ${file}: ${score}%`); + console.log(`Breakdown:`, breakdown); + + if (score >= 95) { + console.log(`✅ Validation passed for ${file}. Updating badge and auto-committing.`); - try { - execFileSync('git', ['add', file]); - try { execFileSync('sh', ['-c', 'git add benchmarks/suites/*.json benchmarks/criteria/*.json 2>/dev/null || true']); } catch (e) {} - // Only commit if there are changes (badge might already be there) - const status = execFileSync('git', ['status', '--porcelain'], { encoding: 'utf-8' }); - if (status.includes(file) || status.includes('benchmarks/')) { - execFileSync('git', ['commit', '-m', '[chore: benchmark-sync]']); - execFileSync('git', ['push', 'origin', 'HEAD:main']); - } else { - console.log(`Badge already present in ${file}, skipping commit.`); + let content = await fsPromises.readFile(file, 'utf-8'); + if (!content.includes('[![Vibe-Coding Verified]')) { + content = content.replace(/^# /, '[![Vibe-Coding Verified](https://img.shields.io/badge/Vibe--Coding-Verified-brightgreen?style=for-the-badge)](#)\n\n# '); + await fsPromises.writeFile(file, content); } - } catch (err) { - console.error('Failed to commit or push:', err.message); - } - } else { - console.error(`❌ Validation failed for ${file}. Score below 95%.`); + try { + execFileSync('git', ['add', file]); + try { execFileSync('sh', ['-c', 'git add benchmarks/suites/*.json benchmarks/criteria/*.json 2>/dev/null || true']); } catch (e) {} + // Only commit if there are changes (badge might already be there) + const status = execFileSync('git', ['status', '--porcelain'], { encoding: 'utf-8' }); + if (status.includes(file) || status.includes('benchmarks/')) { + execFileSync('git', ['commit', '-m', '[chore: benchmark-sync]']); + execFileSync('git', ['push', 'origin', 'HEAD:main']); + } else { + console.log(`Badge already present in ${file}, skipping commit.`); + } + } catch (err) { + console.error('Failed to commit or push:', err.message); + } - const reportDir = path.join('benchmarks', 'logs'); - if (!await fileOrDirExists(reportDir)) await fsPromises.mkdir(reportDir, { recursive: true }); + } else { + console.error(`❌ Validation failed for ${file}. Score below 95%.`); - const reportPath = path.join(reportDir, `violation-report.md`); - const reportContent = `# Critical Violation Report\n\n> [!CAUTION]\n> Fidelity Score dropped below 95%.\n\n**File:** \`${file}\`\n**Fidelity Score:** ${score}%\n**Threshold:** 95%\n\n## Breakdown\n| Metric | Score |\n|---|---|\n| Arch Integrity | ${breakdown.arch} |\n| Type Safety | ${breakdown.type} |\n| Security | ${breakdown.security} |\n| Efficiency | ${breakdown.efficiency} |\n\n## Generated Code\n\`\`\`typescript\n${generatedCode}\n\`\`\`\n\nReview the AST rules.`; + const reportDir = path.join('benchmarks', 'logs'); + if (!await fileOrDirExists(reportDir)) await fsPromises.mkdir(reportDir, { recursive: true }); - await fsPromises.writeFile(reportPath, reportContent); - console.log(`Generated violation report: ${reportPath}`); + const reportPath = path.join(reportDir, `violation-report.md`); + const reportContent = `# Critical Violation Report\n\n> [!CAUTION]\n> Fidelity Score dropped below 95%.\n\n**File:** \`${file}\`\n**Fidelity Score:** ${score}%\n**Threshold:** 95%\n\n## Breakdown\n| Metric | Score |\n|---|---|\n| Arch Integrity | ${breakdown.arch} |\n| Type Safety | ${breakdown.type} |\n| Security | ${breakdown.security} |\n| Efficiency | ${breakdown.efficiency} |\n\n## Generated Code\n\`\`\`typescript\n${generatedCode}\n\`\`\`\n\nReview the AST rules.`; - try { - execFileSync('gh', ['issue', 'create', '--title', `Critical Issue: Fidelity Gap for ${file}`, '--label', 'critical,bug', '--body-file', reportPath]); - console.log(`Created GitHub Issue for ${file}`); - } catch (err) { - console.error('Failed to create GitHub Issue (gh cli might not be installed or authenticated):', err.message); - } + await fsPromises.writeFile(reportPath, reportContent); + console.log(`Generated violation report: ${reportPath}`); - process.exitCode = 1; - } + try { + execFileSync('gh', ['issue', 'create', '--title', `Critical Issue: Fidelity Gap for ${file}`, '--label', 'critical,bug', '--body-file', reportPath]); + console.log(`Created GitHub Issue for ${file}`); + } catch (err) { + console.error('Failed to create GitHub Issue (gh cli might not be installed or authenticated):', err.message); + } + + process.exitCode = 1; + } } } -runVibeCheck().catch(console.error); \ No newline at end of file +if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) { + runVibeCheck().catch(console.error); +} \ No newline at end of file