perf(python): one-pass mirror of the parsed tree; map lookups in linkProject

bin/axiomcode

@@ -324,16 +324,61 @@ case "$cmd" in
     # a caller that checks it is no worse off. Only the languages that produced a graph are
     # listed at the end, so the output stays a list of files that exist.
     solved=(); failed=()
-    for l in "${langs[@]}"; do
-      echo "▶ solving $l → $out/$l/graph.sqlite"
-      if "$0" engine --language "$l" --client-ir "$ir/$l" --out "$out/$l" --intermediate "$int" --library "$libroots" \
-           --meta "source_version=$version" --meta "source_dir=$(cd "$src" && pwd)" ${rest[@]+"${rest[@]}"}; then
-        solved+=("$l"); [ -z "$progress" ] || echo "$l ok $out/$l/graph.sqlite" >> "$progress"
+    # Each language gets its OWN intermediate: run-souffle writes fixed names there
+    # (souffle-program.cpp, .souffle-gen.log), so two concurrent solves into one
+    # directory would overwrite each other's program mid-compile.
+    solve_one(){ local sl="$1"
+      "$0" engine --language "$sl" --client-ir "$ir/$sl" --out "$out/$sl" --intermediate "$int/$sl" --library "$libroots" \
+           --meta "source_version=$version" --meta "source_dir=$(cd "$src" && pwd)" ${rest[@]+"${rest[@]}"}
+    }
+    record(){ local rl="$1" rrc="$2"
+      if [ "$rrc" -eq 0 ]; then
+        solved+=("$rl"); [ -z "$progress" ] || echo "$rl ok $out/$rl/graph.sqlite" >> "$progress"
       else
-        failed+=("$l"); echo "❌ $l failed; continuing with the remaining languages" >&2
-        [ -z "$progress" ] || echo "$l failed" >> "$progress"
+        failed+=("$rl"); echo "❌ $rl failed; continuing with the remaining languages" >&2
+        [ -z "$progress" ] || echo "$rl failed" >> "$progress"
       fi
-    done
+    }
+    # THE FIRST (LARGEST) LANGUAGE SOLVES ALONE, streaming, exactly as before: its graph is the one a
+    # --progress caller publishes first (#1555), and nothing may compete with it for cores or
+    # interleave its log. Only the languages AFTER it run concurrently — they used to wait in line
+    # behind each other for no reason: their solves share nothing (own IR, own output, own
+    # intermediate). Memory is the tradeoff (each is its own Soufflé process), so the width is
+    # modest by default and AXIOMCODE_SOLVE_JOBS raises or lowers it; 1 restores the strict line.
+    l="${langs[0]}"
+    echo "▶ solving $l → $out/$l/graph.sqlite"
+    rc=0; solve_one "$l" || rc=$?; record "$l" "$rc"
+    if [ ${#langs[@]} -gt 1 ]; then
+      others=("${langs[@]:1}")
+      JOBS="${AXIOMCODE_SOLVE_JOBS:-2}"
+      case "$JOBS" in (*[!0-9]*|'') JOBS=2;; esac; [ "$JOBS" -ge 1 ] || JOBS=1
+      if [ "$JOBS" -eq 1 ]; then
+        for l in "${others[@]}"; do
+          echo "▶ solving $l → $out/$l/graph.sqlite"
+          rc=0; solve_one "$l" || rc=$?; record "$l" "$rc"
+        done
+      else
+        # Waves of $JOBS (bash 3.2 has no `wait -n`): start a wave, wait for all of it, replay each
+        # log whole so the build log never interleaves, then record in the wave's order.
+        i=0
+        while [ $i -lt ${#others[@]} ]; do
+          wave=("${others[@]:$i:$JOBS}"); i=$((i + JOBS))
+          wavepids=()
+          for l in "${wave[@]}"; do
+            echo "▶ solving $l → $out/$l/graph.sqlite (concurrent)"
+            { solve_one "$l" > "$int/solve-$l.log" 2>&1; echo $? > "$int/solve-$l.rc"; } &
+            wavepids+=($!)
+          done
+          for wp in "${wavepids[@]}"; do wait "$wp" || true; done
+          for l in "${wave[@]}"; do
+            cat "$int/solve-$l.log" 2>/dev/null
+            rc="$(cat "$int/solve-$l.rc" 2>/dev/null || echo 1)"
+            rm -f "$int/solve-$l.log" "$int/solve-$l.rc"
+            record "$l" "$rc"
+          done
+        done
+      fi
+    fi
     for l in ${solved[@]+"${solved[@]}"}; do echo "$out/$l/graph.sqlite"; done
     if [ ${#failed[@]} -gt 0 ]; then
       echo "❌ ${#failed[@]} of ${#langs[@]} languages failed: ${failed[*]}" >&2

parser/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@axiomcode/parser",
-  "version": "0.1.2",
-  "description": "AxiomCode Parser — compiles source code and build configuration into a relational intermediate representation.",
+  "version": "0.1.3",
+  "description": "AxiomCode Parser \u2014 compiles source code and build configuration into a relational intermediate representation.",
   "main": "dist/extract.js",
   "types": "dist/extract.d.ts",
   "scripts": {

parser/src/parsers/csharp/csharp-parser.ts

@@ -6,6 +6,7 @@ import Parser from 'tree-sitter';
 import CSharp from 'tree-sitter-c-sharp';
 
 import { FILE_EXTENSIONS } from '@/constants/consts';
+import { materializeTree } from '@/parsers/mirror-tree';
 import {
   CSHARP_CALLBACK_PARSE_THRESHOLD,
   CSHARP_PARSE_CHUNK_SIZE,
@@ -81,6 +82,20 @@ export function stripUtf8Bom(sourceCode: string): string {
   return sourceCode.startsWith(UTF8_BOM) ? sourceCode.slice(UTF8_BOM.length) : sourceCode;
 }
 
+/** tree-sitter-c-sharp `extras`: the only node types that parse as extra. */
+const CS_EXTRA_TYPES: ReadonlySet<string> = new Set([
+  'comment',
+  'preproc_region',
+  'preproc_endregion',
+  'preproc_line',
+  'preproc_pragma',
+  'preproc_nullable',
+  'preproc_error',
+  'preproc_warning',
+  'preproc_define',
+  'preproc_undef',
+]);
+
 export class CSharpParser implements LanguageParser {
   readonly language = ProjectLanguage.CSHARP;
   readonly fileExtension = FILE_EXTENSIONS.CSHARP;
@@ -142,7 +157,16 @@ export class CSharpParser implements LanguageParser {
     return parseWithRetry(source);
   }
 
-  getRootNode(tree: Parser.Tree): Parser.SyntaxNode {
+  getRootNode(tree: Parser.Tree, sourceCode?: string): Parser.SyntaxNode {
+    // With `sourceCode`, the root is a one-pass plain-JS mirror of the tree
+    // (see `../mirror-tree`): every stage after it reads JS properties
+    // instead of re-crossing the tree-sitter FFI per property access.
+    // Tree-sitter still parses every file.
+    if (sourceCode !== undefined) {
+      // parse() strips a leading BOM, so the mirror slices text from the SAME
+      // string the tree's byte offsets are relative to, whatever was passed.
+      return materializeTree(tree, stripUtf8Bom(sourceCode), CS_EXTRA_TYPES) as unknown as Parser.SyntaxNode;
+    }
     return tree.rootNode;
   }
 

parser/src/parsers/csharp/extractors/cs-fact-extractor.ts

@@ -216,11 +216,11 @@ export class CsFactExtractor {
     // is. Length-preserving to the character; the receiver the blanking
     // removes is returned in a side table. See cs-extension-block.ts.
     const flattened = flattenExtensionBlocks(rewritten, (source) =>
-      this.parser.getRootNode(this.parser.parse(source))
+      this.parser.getRootNode(this.parser.parse(source), source)
     );
     const parseText = flattened.text.endsWith('\n') ? flattened.text : `${flattened.text}\n`;
     const tree = this.parser.parse(parseText);
-    const root = this.parser.getRootNode(tree);
+    const root = this.parser.getRootNode(tree, parseText);
 
     // The symbol set this emission is compiled under: what the caller supplied,
     // plus the implicit framework symbols the SDK injects and no `.csproj`

parser/src/parsers/java/extractors/import-extractor.ts

@@ -54,7 +54,7 @@ export class ImportExtractor implements BaseExtractor<ImportRegistry> {
       }
 
       const tree = this.javaParser.parse(fileContent);
-      const rootNode = this.javaParser.getRootNode(tree);
+      const rootNode = this.javaParser.getRootNode(tree, fileContent);
 
       this.extractImportsFromRoot(rootNode, filePath, serviceVersionHash, imports);
     } catch (error) {

parser/src/parsers/java/extractors/type-registry-extractor.ts

@@ -203,7 +203,7 @@ export class TypeRegistryExtractor implements BaseExtractor<TypeRegistry> {
       }
 
       const tree = this.javaParser.parse(fileContent);
-      const rootNode = this.javaParser.getRootNode(tree);
+      const rootNode = this.javaParser.getRootNode(tree, fileContent);
 
       const basePath = this.extractBasePath(filePath);
       const fileName = path.basename(filePath);

parser/src/parsers/java/java-parser.ts

@@ -3,9 +3,13 @@ import Java from 'tree-sitter-java';
 
 import { FILE_EXTENSIONS } from '@/constants/consts';
 import { LanguageParser } from '@/parsers/language-parser';
+import { materializeTree } from '@/parsers/mirror-tree';
 import { ProjectLanguage } from '@/types/ProjectInfo';
 import { withRetry } from '@/utils/retry-decorator';
 
+/** tree-sitter-java `extras`: the only node types that parse as extra. */
+const JAVA_EXTRA_TYPES: ReadonlySet<string> = new Set(['line_comment', 'block_comment']);
+
 /**
  * Java-specific tree-sitter parser implementation
  */
@@ -68,11 +72,21 @@ export class JavaParser implements LanguageParser {
   }
 
   /**
-   * Gets the root node of a parsed tree
+   * Gets the root node of a parsed tree.
+   *
+   * With `sourceCode`, the root is a one-pass plain-JS mirror of the tree
+   * (see `../mirror-tree`): every stage after it reads JS properties instead
+   * of re-crossing the tree-sitter FFI per property access. Tree-sitter still
+   * parses every file. Without it, the real tree-sitter root is returned.
+   *
    * @param tree Parsed syntax tree
+   * @param sourceCode The exact string the tree was parsed from
    * @returns Root syntax node
    */
-  getRootNode(tree: Parser.Tree): Parser.SyntaxNode {
+  getRootNode(tree: Parser.Tree, sourceCode?: string): Parser.SyntaxNode {
+    if (sourceCode !== undefined) {
+      return materializeTree(tree, sourceCode, JAVA_EXTRA_TYPES) as unknown as Parser.SyntaxNode;
+    }
     return tree.rootNode;
   }