diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bf608dc..6a4969f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,6 +44,35 @@ jobs: name: vitest needs: pre-commit runs-on: ubuntu-latest + services: + mysql: + image: mysql:8.4 + env: + MYSQL_ROOT_PASSWORD: ci-only-password + MYSQL_ROOT_HOST: '%' + MYSQL_DATABASE: triton_agent_test + ports: + - 3306/tcp + options: >- + --health-cmd="mysqladmin ping --silent" + --health-interval=10s --health-timeout=5s --health-retries=12 + redis: + image: redis:8.2 + ports: + - 6379/tcp + options: >- + --health-cmd="redis-cli ping" + --health-interval=10s --health-timeout=5s --health-retries=6 + rabbitmq: + image: rabbitmq:4.3.6 + env: + RABBITMQ_DEFAULT_USER: ci + RABBITMQ_DEFAULT_PASS: ci-only-password + ports: + - 5672/tcp + options: >- + --health-cmd="rabbitmq-diagnostics -q ping" + --health-interval=10s --health-timeout=5s --health-retries=12 steps: - name: Checkout harness uses: actions/checkout@v4 @@ -67,5 +96,32 @@ jobs: - name: Install and build repository run: ./tools/scripts/install-all.sh + - name: Configure disposable Redis + env: + TRITON_TEST_REDIS_URL: redis://127.0.0.1:${{ job.services.redis.ports['6379'] }}/0 + run: | + packages/dsh-triton-riscv/.venv/bin/python -I - <<'PY' + import os + import redis + + with redis.Redis.from_url(os.environ['TRITON_TEST_REDIS_URL'], + socket_timeout=5, socket_connect_timeout=5, + decode_responses=True) as client: + client.config_set('maxmemory', '64mb') + client.config_set('maxmemory-policy', 'noeviction') + settings = client.config_get('maxmemory*') + assert int(settings['maxmemory']) == 64 * 1024 * 1024, settings + assert settings['maxmemory-policy'] == 'noeviction', settings + PY + + - name: Prepare disposable plugin database + env: + TRITON_MYSQL_URL: mysql+pymysql://root:ci-only-password@127.0.0.1:${{ job.services.mysql.ports['3306'] }}/triton_agent_test + run: packages/dsh-triton-riscv/.venv/bin/python -I -m codex_agent.storage upgrade + - name: Run tests - run: pnpm test:all + env: + TRITON_MYSQL_URL: mysql+pymysql://root:ci-only-password@127.0.0.1:${{ job.services.mysql.ports['3306'] }}/triton_agent_test + TRITON_TEST_REDIS_URL: redis://127.0.0.1:${{ job.services.redis.ports['6379'] }}/0 + TRITON_TEST_AMQP_URL: amqp://ci:ci-only-password@127.0.0.1:${{ job.services.rabbitmq.ports['5672'] }}/ + run: pnpm test:all --reporter=verbose diff --git a/config.yml.example b/config.yml.example index ccf2200..f8c652e 100644 --- a/config.yml.example +++ b/config.yml.example @@ -38,6 +38,14 @@ - id: triton-riscv-native-host config: enabled: false + storage: + urlEnv: TRITON_MYSQL_URL # Secret MySQL URL in the host environment, never in this file + cache: + enabled: false # Optional Redis; requires maxmemory-policy noeviction + urlEnv: TRITON_REDIS_URL + queue: + enabled: false # Start relay/recover/agent/validation/memory processes before enabling + urlEnv: TRITON_AMQP_URL # Optional; defaults to packages/dsh-triton-riscv/.venv/bin/python # python: /absolute/path/to/harness/packages/dsh-triton-riscv/.venv/bin/python permissions: diff --git a/packages/dsh-triton-riscv/README.md b/packages/dsh-triton-riscv/README.md index 2bea3a9..a11f2bc 100644 --- a/packages/dsh-triton-riscv/README.md +++ b/packages/dsh-triton-riscv/README.md @@ -8,6 +8,7 @@ A Harness plugin for operator development, testing, repair, and historical evide node >= 22.19 (with npm) pnpm >= 11 python3 >= 3.10 +MySQL 8.4 ``` An existing Triton-RISCV checkout is required as the workspace. @@ -26,6 +27,17 @@ Follow the [host configuration](../../README.md). Set `config.enabled: true` in ./tools/scripts/install-all.sh ``` +Create an empty MySQL database named `triton_agent`. Export its connection URL +in the terminal that starts Harness (URL-encode special characters in passwords): + +```sh +export TRITON_MYSQL_URL='mysql+pymysql://USER:PASSWORD@127.0.0.1:3306/triton_agent' +packages/dsh-triton-riscv/.venv/bin/python -m codex_agent.storage upgrade +``` + +Keep credentials outside Git. Each selected workspace has isolated records. +For existing SQLite data, [import it before first launch](python/codex_agent/README.md#existing-data). + ### Quick Activate Set `DEEPSEEK_API_KEY` in your terminal environment, then run from the repository root: @@ -64,13 +76,47 @@ Open **http://127.0.0.1:8765**. FastAPI serves the built frontend; no separate f Live requests need separate `ISRC_API_KEY` and `DSH_MODEL` settings. Workbench and native sessions are independent. +### Optional Redis Cache + +Use a private standalone Redis with a memory limit and `maxmemory-policy noeviction`. +Export `TRITON_REDIS_URL` in the Harness terminal; set `cache.enabled: true` in the +plugin config. Credentials stay outside Git. Redis is off by default. + +```sh +export TRITON_REDIS_URL='redis://127.0.0.1:6379/0' +``` + +Historical retrieval, case details and statistics are cached; approvals and run +state are not. Writes invalidate versioned results. Cache errors use bounded +MySQL fallback, then report busy rather than returning false "not found" results. +All workers must use the same limits and MySQL endpoint. This is not multi-user authentication. +See [backend options](python/codex_agent/README.md#redis-options). + ### Development Tests -From the plugin directory: +Vitest drives JavaScript, Python and native-host integration tests. After setup, +from the plugin directory, configure **disposable** services (never production): ```sh +export TRITON_MYSQL_URL='mysql+pymysql://USER:PASSWORD@127.0.0.1:3306/triton_agent_test' +export TRITON_TEST_REDIS_URL='redis://127.0.0.1:6379/0' +export TRITON_TEST_AMQP_URL='amqp://guest:guest@127.0.0.1:5672/' +.venv/bin/python -I -m codex_agent.storage upgrade npm test -.venv/bin/python -m unittest discover -s python/codex_agent/tests -v +``` + +Use `npm run test:unit` for JavaScript-only checks or `npm run test:backend` for +Python tests through Vitest. Python failures, collection errors and timeouts fail +the Vitest run. Model API keys and a RISC-V server are not required. CI provisions +its own service containers; private-service restart tests need the optional +controls described in the backend README and otherwise report skips. + +Before pushing, run from the repository root: + +```sh +python3 -m pip install pre-commit +pre-commit run --all-files +pnpm test:all ``` For trusted local use only. Never commit credentials. Backend development commands are in [python/codex_agent/README.md](python/codex_agent/README.md). diff --git a/packages/dsh-triton-riscv/frontend/src/App.tsx b/packages/dsh-triton-riscv/frontend/src/App.tsx index 22b5035..896e2d8 100644 --- a/packages/dsh-triton-riscv/frontend/src/App.tsx +++ b/packages/dsh-triton-riscv/frontend/src/App.tsx @@ -305,6 +305,7 @@ export default function App() { } | null>(null) const eventSource = useRef(null) const messagesEnd = useRef(null) + const pendingRequest = useRef<{ session: string; text: string; id: string } | null>(null) const closeStream = () => { eventSource.current?.close() @@ -470,7 +471,12 @@ export default function App() { setStreamingText('') setError(null) try { - const result = await api.sendMessage(activeSessionId, content.trim()) + const text = content.trim() + if (pendingRequest.current?.session !== activeSessionId || pendingRequest.current.text !== text) { + pendingRequest.current = { session: activeSessionId, text, id: crypto.randomUUID() } + } + const result = await api.sendMessage(activeSessionId, text, pendingRequest.current.id) + pendingRequest.current = null setBundle(current => current ? { diff --git a/packages/dsh-triton-riscv/frontend/src/api.test.ts b/packages/dsh-triton-riscv/frontend/src/api.test.ts index a1256ec..a9bcbe5 100644 --- a/packages/dsh-triton-riscv/frontend/src/api.test.ts +++ b/packages/dsh-triton-riscv/frontend/src/api.test.ts @@ -18,13 +18,13 @@ describe('FastAPI client', () => { const fetchMock = vi.fn().mockResolvedValue(jsonResponse({ run: null })) vi.stubGlobal('fetch', fetchMock) - await api.sendMessage('session-1', '验证 relu_and_mul') + await api.sendMessage('session-1', '验证 relu_and_mul', 'request-1') expect(fetchMock).toHaveBeenCalledWith( '/api/sessions/session-1/messages', expect.objectContaining({ method: 'POST', - body: JSON.stringify({ content: '验证 relu_and_mul' }), + body: JSON.stringify({ content: '验证 relu_and_mul', request_id: 'request-1' }), }), ) }) diff --git a/packages/dsh-triton-riscv/frontend/src/api.ts b/packages/dsh-triton-riscv/frontend/src/api.ts index fb46a06..321f8dc 100644 --- a/packages/dsh-triton-riscv/frontend/src/api.ts +++ b/packages/dsh-triton-riscv/frontend/src/api.ts @@ -46,10 +46,10 @@ export const api = { } }, getSessionApprovals: (sessionId: string) => request(`/api/sessions/${sessionId}/approvals`), - sendMessage: (sessionId: string, content: string) => + sendMessage: (sessionId: string, content: string, requestId: string = crypto.randomUUID()) => request(`/api/sessions/${sessionId}/messages`, { method: 'POST', - body: JSON.stringify({ content }), + body: JSON.stringify({ content, request_id: requestId }), }), getRun: (runId: string) => request(`/api/runs/${runId}`), getEvents: (runId: string) => request(`/api/runs/${runId}/events`), diff --git a/packages/dsh-triton-riscv/lib/config.js b/packages/dsh-triton-riscv/lib/config.js index 93986db..91979ed 100644 --- a/packages/dsh-triton-riscv/lib/config.js +++ b/packages/dsh-triton-riscv/lib/config.js @@ -31,17 +31,117 @@ export const CONFIG_ENV = 'TRITON_RISCV_CONFIG' export function resolveConfig(input = {}) { const c = object( input, - ['enabled', 'repoRoot', 'python', 'stateDir', 'permissions', 'remote', 'memory'], + ['enabled', 'repoRoot', 'python', 'stateDir', 'permissions', 'remote', 'memory', 'storage', 'cache', 'queue'], 'triton-riscv config', ) const p = object(c.permissions, ['validation', 'development', 'repair'], 'permissions') const r = object(c.remote, ['host', 'repository', 'required', 'requireTaskQuotas'], 'remote') - const m = object(c.memory, ['database', 'retrievalMode', 'contextFormat', 'embedding'], 'memory') + const m = object(c.memory, ['retrievalMode', 'contextFormat', 'embedding'], 'memory') const e = object( m.embedding, ['provider', 'model', 'baseUrl', 'tokenizerJson', 'tokenBudget', 'apiKeyEnv'], 'embedding', ) + const db = object(c.storage, ['urlEnv', 'poolSize', 'maxOverflow', 'poolTimeout'], 'storage') + const cacheDefaults = { + enabled: false, + urlEnv: 'TRITON_REDIS_URL', + ttlSeconds: 60, + negativeTtlSeconds: 20, + lockMs: 10000, + waitMs: 1000, + maxBytes: 262144, + maxConcurrent: 4, + requestsPerSecond: 100, + rebuildsPerSecond: 10, + bloom: false, + bloomCapacity: 10000, + bloomErrorRate: 0.001, + } + const cache = { ...cacheDefaults, ...object(c.cache, Object.keys(cacheDefaults), 'cache') } + for (const name of ['enabled', 'bloom']) cache[name] = boolean(cache[name], false, 'cache.' + name) + for (const [key, [min, max]] of Object.entries({ + ttlSeconds: [1, 3600], + negativeTtlSeconds: [1, 60], + lockMs: [100, 60000], + waitMs: [10, 5000], + maxBytes: [1024, 1048576], + maxConcurrent: [1, 32], + requestsPerSecond: [1, 10000], + rebuildsPerSecond: [1, 1000], + bloomCapacity: [100, 10000000], + })) { + if (!Number.isInteger(cache[key]) || cache[key] < min || cache[key] > max) throw new Error('Invalid cache.' + key) + } + if ( + typeof cache.bloomErrorRate !== 'number' || + !Number.isFinite(cache.bloomErrorRate) || + cache.bloomErrorRate <= 0 || + cache.bloomErrorRate > 0.1 + ) + throw new Error('Invalid cache.bloomErrorRate') + cache.urlEnv = text(cache.urlEnv, 'TRITON_REDIS_URL', 'cache.urlEnv') + if ( + !/^[A-Za-z_][A-Za-z0-9_]*$/.test(cache.urlEnv) || + /^(TRITON_RISCV_|RISCV_)/.test(cache.urlEnv) || + ['PATH', 'HOME', 'PYTHONPATH', 'PYTHONHOME', db.urlEnv ?? 'TRITON_MYSQL_URL'].includes(cache.urlEnv) + ) + throw new Error('Invalid cache.urlEnv') + const urlEnv = text(db.urlEnv, 'TRITON_MYSQL_URL', 'storage.urlEnv') + if ( + !/^[A-Za-z_][A-Za-z0-9_]*$/.test(urlEnv) || + /^(TRITON_RISCV_|RISCV_)/.test(urlEnv) || + ['PATH', 'HOME', 'PYTHONPATH', 'PYTHONHOME'].includes(urlEnv) + ) + throw new Error('Invalid storage.urlEnv') + const storage = { + urlEnv, + poolSize: db.poolSize ?? 5, + maxOverflow: db.maxOverflow ?? 5, + poolTimeout: db.poolTimeout ?? 10, + } + const queueDefaults = { + enabled: false, + urlEnv: 'TRITON_AMQP_URL', + leaseSeconds: 60, + maxAttempts: 3, + jobTimeoutSeconds: 1200, + maxPending: 100, + } + const queue = { ...queueDefaults, ...object(c.queue, Object.keys(queueDefaults), 'queue') } + queue.enabled = boolean(queue.enabled, false, 'queue.enabled') + queue.urlEnv = text(queue.urlEnv, 'TRITON_AMQP_URL', 'queue.urlEnv') + if ( + !/^[A-Za-z_][A-Za-z0-9_]*$/.test(queue.urlEnv) || + /^(TRITON_RISCV_|RISCV_)/.test(queue.urlEnv) || + [ + 'PATH', + 'HOME', + 'PYTHONPATH', + 'PYTHONHOME', + storage.urlEnv, + cache.urlEnv, + e.apiKeyEnv ?? 'AGENT_EMBEDDING_API_KEY', + ].includes(queue.urlEnv) + ) + throw new Error('Invalid queue.urlEnv') + for (const [key, [min, max]] of Object.entries({ + leaseSeconds: [15, 600], + maxAttempts: [1, 10], + jobTimeoutSeconds: [30, 7200], + maxPending: [1, 10000], + })) { + if (!Number.isInteger(queue[key]) || queue[key] < min || queue[key] > max) throw new Error('Invalid queue.' + key) + } + for (const [key, value] of Object.entries(storage)) { + if (key === 'urlEnv') continue + if ( + !Number.isInteger(value) || + value < (key === 'maxOverflow' ? 0 : 1) || + value > (key === 'poolTimeout' ? 60 : 50) + ) + throw new Error('Invalid storage.' + key) + } const enabled = boolean(c.enabled, false, 'enabled') const repoRoot = absolute(c.repoRoot, '', 'repoRoot') const python = absolute(c.python, join(packageRoot, '.venv/bin/python'), 'python') @@ -69,6 +169,9 @@ export function resolveConfig(input = {}) { repoRoot, python, stateDir, + storage, + cache, + queue, permissions: { validation: boolean(p.validation, false, 'permissions.validation'), development: boolean(p.development, false, 'permissions.development'), @@ -76,7 +179,6 @@ export function resolveConfig(input = {}) { }, remote: { host, repository, required, requireTaskQuotas }, memory: { - database: absolute(m.database, stateDir ? join(stateDir, 'memory.sqlite3') : '', 'memory.database'), retrievalMode: text(m.retrievalMode, 'legacy', 'memory.retrievalMode'), contextFormat: text(m.contextFormat, 'classic', 'memory.contextFormat'), embedding: { @@ -97,6 +199,12 @@ export function workerEnvironment(config, ambient = process.env) { const env = { [CONFIG_ENV]: document } const key = config.memory.embedding.apiKeyEnv if (ambient[key]) env[key] = ambient[key] + const databaseKey = config.storage.urlEnv + if (ambient[databaseKey]) env[databaseKey] = ambient[databaseKey] + const cacheKey = config.cache.urlEnv + if (config.cache.enabled && ambient[cacheKey]) env[cacheKey] = ambient[cacheKey] + const queueKey = config.queue.urlEnv + if (config.queue.enabled && ambient[queueKey]) env[queueKey] = ambient[queueKey] return env } export function bridgeEnvironment(config, ambient = process.env) { @@ -141,7 +249,6 @@ export function configFromEnvironment(env) { requireTaskQuotas: quota === '1' || quota === 'true', }, memory: { - database: env.TRITON_RISCV_MEMORY_DB, retrievalMode: env.TRITON_RISCV_MEMORY_RETRIEVAL_MODE, contextFormat: env.TRITON_RISCV_MEMORY_CONTEXT_FORMAT, embedding: { diff --git a/packages/dsh-triton-riscv/lib/native-adapter.js b/packages/dsh-triton-riscv/lib/native-adapter.js index 35d71f2..2ac860e 100644 --- a/packages/dsh-triton-riscv/lib/native-adapter.js +++ b/packages/dsh-triton-riscv/lib/native-adapter.js @@ -187,6 +187,7 @@ export function installNativeAdapter(ctx, { bridge = callBridge, store = createS if (name === 'get_operator_implementation_proposal') own(state, args.proposal_id, 'development') if (name === 'retrieve_operator_memory' && args.run_id) own(state, args.run_id, 'run') if (name === 'get_validation_job') own(state, args.job_id, 'job') + if (name === 'inspect_queued_task') own(state, args.job_id, 'async-job') if (name === 'get_validation_status') own(state, args.run_id, 'plan') const kind = APPLY[name] || (name === 'validate_operator' && args.execute === true ? 'validation' : null) if (kind) { @@ -272,8 +273,29 @@ export function installNativeAdapter(ctx, { bridge = callBridge, store = createS exec.signal.addEventListener('abort', cancelListener, { once: true }) } const result = await next() - const data = decode(result) + let data = decode(result) if (!data || typeof data !== 'object') return result + if ( + ['validate_operator', 'execute_approved_validation'].includes(name) && + (data.async_job || data.status === 'queued') && + data.job_id + ) { + state.artifacts[data.job_id] = 'async-job' + state.task.async_job_id = data.job_id + state.task.validation_status = data.queue_status || data.status + store.save(state) + return result + } + const inspectedValidation = + name === 'inspect_queued_task' && + state.task.async_job_id === args.job_id && + data.kind === 'validation' && + ['succeeded', 'failed'].includes(data.status) && + data.result?.receipt_path + if (name === 'inspect_queued_task') { + if (!inspectedValidation) return result + data = data.result + } if (name === 'get_operator_implementation_proposal') return result if (name === 'get_validation_status') return result if (name === 'get_validation_job' && state.active.job !== data.job_id) return result @@ -321,7 +343,7 @@ export function installNativeAdapter(ctx, { bridge = callBridge, store = createS run_id: item.run_id, })) } - if (['validate_operator', 'execute_approved_validation'].includes(name) && data.run_id) { + if ((['validate_operator', 'execute_approved_validation'].includes(name) || inspectedValidation) && data.run_id) { state.artifacts[data.run_id] = data.status === 'planned' ? 'plan' : 'run' if (data.status !== 'planned') state.artifact_requests[data.run_id] = @@ -344,7 +366,7 @@ export function installNativeAdapter(ctx, { bridge = callBridge, store = createS state.active = { request: args.development_id, development: data.proposal_id } if (name === 'propose_repair' && data.proposal_id) state.active = { run: args.run_id, repair: data.proposal_id } if (name === 'prepare_validation_job' && data.job_id) state.active = { job: data.job_id } - if (['validate_operator', 'execute_approved_validation'].includes(name) && data.run_id) { + if ((['validate_operator', 'execute_approved_validation'].includes(name) || inspectedValidation) && data.run_id) { if (data.status === 'planned') state.active = { plan: data.run_id } else state.active.run = data.run_id } @@ -356,7 +378,7 @@ export function installNativeAdapter(ctx, { bridge = callBridge, store = createS delete state.active.plan for (const key of ['run_id', 'receipt_path', 'log_path', 'exit_code', 'failure_stage']) delete state.task[key] state.task.validation_status = 'not_validated_after_change' - } else if (['validate_operator', 'execute_approved_validation'].includes(name)) { + } else if (['validate_operator', 'execute_approved_validation'].includes(name) || inspectedValidation) { state.task.validation_status = data.status } diff --git a/packages/dsh-triton-riscv/lib/native-workspace.js b/packages/dsh-triton-riscv/lib/native-workspace.js index 7aa0baf..4221dce 100644 --- a/packages/dsh-triton-riscv/lib/native-workspace.js +++ b/packages/dsh-triton-riscv/lib/native-workspace.js @@ -1,6 +1,6 @@ import { createHash } from 'node:crypto' import { realpathSync, statSync } from 'node:fs' -import { basename, dirname, isAbsolute, join } from 'node:path' +import { isAbsolute, join } from 'node:path' import { mcpConfiguration, resolveConfig } from './config.js' import { installNativeAdapter } from './native-adapter.js' import { callBridge } from './native-bridge.js' @@ -33,12 +33,7 @@ export function resolveSessionConfig(input, session) { ? base.stateDir : join(base.stateDir, 'workspaces', id) : join(root, 'agent-results') - const database = input.memory?.database - ? legacy - ? base.memory.database - : join(dirname(base.memory.database), 'workspaces', id, basename(base.memory.database)) - : join(stateDir, 'memory.sqlite3') - return resolveConfig({ ...input, repoRoot: root, stateDir, memory: { ...input.memory, database } }) + return resolveConfig({ ...input, repoRoot: root, stateDir }) } export function installWorkspaceHost(ctx, input, { mcp, createScope }) { diff --git a/packages/dsh-triton-riscv/lib/tool-policy.js b/packages/dsh-triton-riscv/lib/tool-policy.js index 7509214..e04f74a 100644 --- a/packages/dsh-triton-riscv/lib/tool-policy.js +++ b/packages/dsh-triton-riscv/lib/tool-policy.js @@ -10,6 +10,7 @@ export const OPERATOR_TOOLS = new Set( 'execute_validation_job', 'get_validation_job', 'get_validation_status', + 'inspect_queued_task', 'evaluate_plugin_fixtures', 'retrieve_operator_memory', 'execute_approved_validation', diff --git a/packages/dsh-triton-riscv/package.json b/packages/dsh-triton-riscv/package.json index cdc0252..6c61729 100644 --- a/packages/dsh-triton-riscv/package.json +++ b/packages/dsh-triton-riscv/package.json @@ -30,6 +30,7 @@ "python/codex_agent/**/*.json", "python/codex_agent/**/*.md", "python/codex_agent/**/*.yml", + "python/codex_agent/tests/fixtures/*.sql", "python/codex_agent/**/requirements.txt", "python/codex_agent/harness/bundle/index.js", "python/codex_agent/harness/bundle/native.js", @@ -61,6 +62,8 @@ "test:native-host": "node scripts/test-native-host.mjs", "check:package": "node scripts/check-package.mjs", "test": "vitest run --config vitest.config.ts", + "test:unit": "vitest run --config vitest.config.ts tests/unit", + "test:backend": "vitest run --config vitest.config.ts tests/integration/python.spec.ts", "prepack": "npm run build && npm test && node scripts/check-package.mjs" }, "devDependencies": { diff --git a/packages/dsh-triton-riscv/policy.md b/packages/dsh-triton-riscv/policy.md index 2c257f0..2002897 100644 --- a/packages/dsh-triton-riscv/policy.md +++ b/packages/dsh-triton-riscv/policy.md @@ -30,6 +30,10 @@ For an existing operator: 4. Call `mcp__triton_riscv__validate_operator` with `execute=false` to plan. 5. Call `mcp__triton_riscv__execute_approved_validation` with the exact plan's `run_id` to request host approval and execution. +When queue mode returns `status=queued` and a `job_id`, report submission, not success. +End the turn so the validation worker can take the workspace. On a later turn, +use `mcp__triton_riscv__inspect_queued_task`; do not poll in a loop or create another plan. + If remote validation loses its SSH connection, keep the original approved `run_id`. Calling `execute_approved_validation` again with that exact ID can query/collect the original durable remote job; it must not start a replacement. diff --git a/packages/dsh-triton-riscv/prompts/1-skills.md b/packages/dsh-triton-riscv/prompts/1-skills.md index 973c779..d34319e 100644 --- a/packages/dsh-triton-riscv/prompts/1-skills.md +++ b/packages/dsh-triton-riscv/prompts/1-skills.md @@ -8,6 +8,10 @@ For an existing operator: 4. Call `mcp__triton_riscv__validate_operator` with `execute=false` to plan. 5. Call `mcp__triton_riscv__execute_approved_validation` with the exact plan's `run_id` to request host approval and execution. +When queue mode returns `status=queued` and a `job_id`, report submission, not success. +End the turn so the validation worker can take the workspace. On a later turn, +use `mcp__triton_riscv__inspect_queued_task`; do not poll in a loop or create another plan. + If remote validation loses its SSH connection, keep the original approved `run_id`. Calling `execute_approved_validation` again with that exact ID can query/collect the original durable remote job; it must not start a replacement. diff --git a/packages/dsh-triton-riscv/python/codex_agent/README.md b/packages/dsh-triton-riscv/python/codex_agent/README.md index 673eadf..3f84f1a 100644 --- a/packages/dsh-triton-riscv/python/codex_agent/README.md +++ b/packages/dsh-triton-riscv/python/codex_agent/README.md @@ -10,11 +10,127 @@ From `packages/dsh-triton-riscv/` after plugin setup: ```sh .venv/bin/python -m pip install -e './python[workbench]' -.venv/bin/python -m unittest discover -s python/codex_agent/tests -v +.venv/bin/python -m codex_agent.storage upgrade +npm run test:backend ``` To inspect a standalone command, run `.venv/bin/python -I -m codex_agent.operator_agent --help`. Use `-I` with installed CLI tools to avoid importing an older backend from the target checkout. Generated operators and validation artifacts belong to the target checkout. Credentials, databases, and execution logs are not package source files. -To copy historical memory, inspect the options with `.venv/bin/python -I -m codex_agent.migrate_memory --help`. Keep the original data until the copied evidence has been checked. +MySQL 8.4 and `TRITON_MYSQL_URL` are required; use disposable services for tests +as shown in the [plugin README](../../README.md#development-tests). Vitest runs each +Python test module with pytest, including both `unittest.TestCase` and function +tests. It reports Python failures and skips; no separate Python CI job is needed. +`storage.urlEnv` in the plugin config can name a different credential variable. +Schema changes are explicit; startup never migrates data or falls back to SQLite. + +## Existing Data + +Stop writers and back up the old databases first. After the schema upgrade, +import into a workspace that has not yet been opened in MySQL: + +```sh +.venv/bin/python -m codex_agent.storage.import_sqlite \ + --workspace /absolute/path/to/triton-riscv \ + --platform /path/to/platform.sqlite3 \ + --memory /path/to/memory.sqlite3 +``` + +Omit absent sources. Import each legacy reference catalog separately with +`--workspace /absolute/path/to/library --references /path/to/catalog.sqlite3`. +All supplied sources commit together after field/count checks; existing target +workspaces are rejected. IDs, labels, source paths and message order are preserved. +Older chunk layouts are rebuilt and their stale vectors cleared; use +`codex_agent.memory --workspace PATH embed-missing --help` to re-index if needed. + +Evidence files stay in place. Retain them and the original databases for rollback; +the importer never reclassifies historical results as verified successes. +`MemoryStore` and `PlatformStore` now take a workspace directory, not a database +file. The memory CLI uses `--workspace`; development uses `--memory-workspace`. + +## Redis Options + +Configure `cache` alongside `storage` in the plugin config. CLI/workbench processes +can use the same versioned `TRITON_RISCV_CONFIG` JSON document; the URL remains in +`TRITON_REDIS_URL` (or the name in `cache.urlEnv`). Restart workers after changing it. + +Defaults: `ttlSeconds: 60` and `negativeTtlSeconds: 20` (both jittered by 20%), +`lockMs: 10000`, `waitMs: 1000`, `maxBytes: 262144`, `maxConcurrent: 4`, +`requestsPerSecond: 100`, `rebuildsPerSecond: 10`. Rates are per workspace/domain +per second, with a database-wide cap of four times each rate. Use identical limits +across workers. Redis needs script commands and read-only `CONFIG GET` permission. + +Redis loss opens a two-second circuit breaker. Fallback is limited to four attempts +per second and two callers per process; MySQL connection-owned slots also enforce +the configured cross-process rebuild concurrency. Cache locks never authorize writes. + +Bloom is optional (`bloom: true`, `bloomCapacity: 10000`, `bloomErrorRate: 0.001`) +and requires `BF.*` support. With the same runtime config, run: + +```sh +.venv/bin/python -I -m codex_agent.memory --workspace /path/to/checkout warm-cache-ids +.venv/bin/python -I -m codex_agent.memory --workspace /path/to/checkout show 1 +``` + +The filter only applies to exact case IDs, never natural-language retrieval. +It expires after ten minutes and is bypassed after data changes until rebuilt. +This version supports standalone Redis, not Redis Cluster. `allkeys-lfu/lru` are +rejected because the instance also holds locks. Keep services private; use ACLs/TLS +for remote connections. Do not give database or cache credentials to generated tests. + +For integration tests, configure all three disposable services from the plugin +README, upgrade the schema, then run `npm run test:backend`. +`TRITON_TEST_REDIS_SERVER` enables the real stop/restart test. These tests change +Redis memory settings; never target a shared service. Cache metrics are available +through `store.cache.backend.snapshot()` and are process-local, not a monitoring service. + +## RabbitMQ Workers + +Queue mode is opt-in. Install the updated backend and run `codex_agent.storage upgrade` +against a backed-up MySQL database. Set `TRITON_AMQP_URL` to a private RabbitMQ +connection URL. Use the same versioned config, credentials and checkout in every process: + +```sh +export TRITON_RISCV_CONFIG='{"schemaVersion":1,"repoRoot":"/path/to/checkout","stateDir":"/path/to/checkout/agent-results","permissions":{"validation":true},"queue":{"enabled":true}}' +.venv/bin/python -I -m codex_agent.storage upgrade +.venv/bin/python -I -m codex_agent.platform --host 127.0.0.1 --port 8765 +``` + +In separate terminals (or a process supervisor), start each role: + +```sh +.venv/bin/python -I -m codex_agent.workers relay --workspace /path/to/checkout +.venv/bin/python -I -m codex_agent.workers recover --workspace /path/to/checkout +.venv/bin/python -I -m codex_agent.workers agent --workspace /path/to/checkout +.venv/bin/python -I -m codex_agent.workers validation --workspace /path/to/checkout +.venv/bin/python -I -m codex_agent.workers memory --workspace /path/to/checkout +``` + +The workbench persists messages/jobs/outbox together and returns HTTP 202. Clients +must reuse `request_id` on retries. Read `/api/jobs/{id}` for attempts, delivery +status and artifact links; `POST /api/jobs/{id}/cancel` requests cancellation. +`POST /api/jobs/validation` accepts `operator`, `approved_run_id`, `request_id`. +It never grants approval. Native `execute_approved_validation` queues work when enabled; +`inspect_queued_task` reads its result on a later turn. Batch/project tools remain synchronous. + +Defaults: 60-second worker leases, 3 safe attempts, 1,200-second job deadline, +100 outstanding user jobs per workspace, one delivery per worker. Mutating jobs +are serialized per workspace. Memory ingestion follows audited validation, not broker ACK. +Broker confirms mean "delivered to broker", not "tests passed". Delivery is at least once. + +An interrupted effectful task becomes `needs-reconciliation`, never an automatic +new model/SSH run. After inspecting and stopping any remaining processes, an operator +can close it with `workers reconcile --workspace PATH --job ID --note "checks performed" +--confirm-no-active-execution`. This records an attestation, not a verified success; +existing lifecycle journals still guard later execution. + +This release uses separate local processes, durable classic queues and local +content-addressed artifacts. API/workers must share the same host, absolute checkout +and state path. It is not a multi-host HA deployment: shared object storage, quorum +queues, authentication, monitoring and retention policies remain deployment work. +Redis is never the task source of truth. Keep all services private. + +With the same disposable service configuration, run just the queue module: +`npm run test:backend -- -t test_task_queue.py`. +The optional private-broker restart test also needs `TRITON_TEST_RABBITMQ_CTL`. diff --git a/packages/dsh-triton-riscv/python/codex_agent/artifacts/__init__.py b/packages/dsh-triton-riscv/python/codex_agent/artifacts/__init__.py new file mode 100644 index 0000000..92747b4 --- /dev/null +++ b/packages/dsh-triton-riscv/python/codex_agent/artifacts/__init__.py @@ -0,0 +1,63 @@ +"""Content-addressed task artifacts; the current adapter uses the workspace filesystem.""" +import hashlib +import os +from pathlib import Path +import re +import tempfile + +from codex_agent.paths import state_root +from codex_agent.tasks.schema import artifacts +from codex_agent.tasks.store import digest + + +class ArtifactStore: + def __init__(self, root): + self.folder = (state_root(root) / "task-artifacts").resolve() + + def put(self, jobs, job_id, owner, name, data): + if not re.fullmatch(r"[A-Za-z0-9_.-]{1,120}", name) or len(data) > 16 * 1024 * 1024: + raise ValueError("invalid artifact name or artifact exceeds 16 MiB") + checksum = hashlib.sha256(data).hexdigest() + self.folder.mkdir(parents=True, exist_ok=True) + target = self.folder / checksum + with tempfile.NamedTemporaryFile(dir=self.folder, delete=False) as stream: + temp = Path(stream.name) + try: + stream.write(data) + stream.flush() + os.fsync(stream.fileno()) + except BaseException: + temp.unlink(missing_ok=True) + raise + try: + os.replace(temp, target) + directory = os.open(self.folder, os.O_RDONLY) + try: + os.fsync(directory) + finally: + os.close(directory) + with jobs.db.transaction() as conn: + job = jobs.require_owner(conn, job_id, owner) + identifier = digest([job_id, job["attempt"], name]) + old = jobs.db.rows(artifacts, artifacts.c.id == identifier, connection=conn) + if old: + if old[0]["sha256"] != checksum: + raise ValueError("artifact identity already has different bytes") + else: + jobs.db.insert(conn, artifacts, dict(id=identifier, job_id=job_id, attempt=job["attempt"], + sha256=checksum, size=len(data), name=name, storage_key=checksum)) + return identifier + finally: + temp.unlink(missing_ok=True) + + def read(self, record): + key = record["storage_key"] + if not re.fullmatch(r"[0-9a-f]{64}", key): + raise ValueError("invalid artifact storage key") + path = self.folder / key + if path.is_symlink() or path.stat().st_size != record["size"]: + raise ValueError("artifact changed") + data = path.read_bytes() + if hashlib.sha256(data).hexdigest() != record["sha256"]: + raise ValueError("artifact checksum mismatch") + return data diff --git a/packages/dsh-triton-riscv/python/codex_agent/core/ports.py b/packages/dsh-triton-riscv/python/codex_agent/core/ports.py index de5a651..342fdda 100644 --- a/packages/dsh-triton-riscv/python/codex_agent/core/ports.py +++ b/packages/dsh-triton-riscv/python/codex_agent/core/ports.py @@ -50,7 +50,7 @@ def invoke(self, request: ModelRequest) -> ModelResponse: class MemoryPort(Protocol): - """Port implemented by SQLite memory and future remote/vector stores.""" + """Port implemented by the workspace-scoped MySQL memory store.""" def retrieve(self, query: Any, limit: int = 5) -> list[dict]: """Retrieve evidence records for one context.""" diff --git a/packages/dsh-triton-riscv/python/codex_agent/demo.py b/packages/dsh-triton-riscv/python/codex_agent/demo.py index adfd386..db6a460 100644 --- a/packages/dsh-triton-riscv/python/codex_agent/demo.py +++ b/packages/dsh-triton-riscv/python/codex_agent/demo.py @@ -134,7 +134,7 @@ def memory_demo(memory_path: Path, development_dir: Path) -> dict: ) stats = store.stats() return { - "database": memory_path.as_posix(), + "database": "mysql", "workspace": memory_path.as_posix(), "ingestion": ingestion, "stats": stats, "query": { @@ -321,7 +321,7 @@ def main() -> int: repo_root / "agent-results" / "development" ), "memory": memory_demo( - repo_root / "agent-results" / "memory.sqlite3", + repo_root, repo_root / "agent-results" / "development", ), "project_discovery": project_demo( diff --git a/packages/dsh-triton-riscv/python/codex_agent/develop_operator.py b/packages/dsh-triton-riscv/python/codex_agent/develop_operator.py index c654904..f768262 100644 --- a/packages/dsh-triton-riscv/python/codex_agent/develop_operator.py +++ b/packages/dsh-triton-riscv/python/codex_agent/develop_operator.py @@ -1313,7 +1313,7 @@ def parse_args() -> argparse.Namespace: default=True, help="Retrieve and record evidence-gated operator memories.", ) - parser.add_argument("--memory-db", default="agent-results/memory.sqlite3") + parser.add_argument("--memory-workspace", type=Path, help="Isolated MySQL memory workspace; defaults to the target repository") parser.add_argument("--memory-limit", type=int, default=5) parser.add_argument( "--memory-auto-ingest", @@ -1361,7 +1361,7 @@ def main() -> int: memory_ingest = None generation_memories: list[dict] = [] if args.memory: - memory_path = Path(args.memory_db) + memory_path = args.memory_workspace or repo_root if not memory_path.is_absolute(): memory_path = repo_root / memory_path try: @@ -1407,7 +1407,7 @@ def main() -> int: run_dir / "memory-status.json", { "enabled": args.memory, - "database": args.memory_db if args.memory else None, + "database": "mysql" if args.memory else None, "embedding_provider": args.embedding_provider, "auto_ingest": memory_ingest, "retrieved": len(generation_memories), diff --git a/packages/dsh-triton-riscv/python/codex_agent/diagnostic_memory.py b/packages/dsh-triton-riscv/python/codex_agent/diagnostic_memory.py index dce8bf5..1660de5 100644 --- a/packages/dsh-triton-riscv/python/codex_agent/diagnostic_memory.py +++ b/packages/dsh-triton-riscv/python/codex_agent/diagnostic_memory.py @@ -22,19 +22,11 @@ ) -MEMORY_DB = Path("agent-results/memory.sqlite3") MAX_MEMORY_RESULTS = 10 -def memory_database_path(repo_root: Path) -> Path: - from codex_agent.paths import state_root - value = runtime_config().memory.database - if value is None: - return state_root(repo_root) / "memory.sqlite3" - configured = Path(value) - if configured.is_absolute(): - return configured - return repo_root.resolve() / configured +def memory_workspace(repo_root: Path) -> Path: + return repo_root.resolve() def build_configured_embedding_provider(): @@ -185,7 +177,7 @@ def remember_validation( if record is None: return {"status": "not-recorded", "reason": "validation was not executed"} provider = build_configured_embedding_provider() - with MemoryStore(memory_database_path(repo_root), provider) as store: + with MemoryStore(memory_workspace(repo_root), provider) as store: memory_id, created = store.add(record) return { "status": "recorded" if created else "deduplicated", @@ -277,7 +269,7 @@ def retrieve_memories( diagnostic_text = " ".join([rule_id or "", *evidence]) try: provider = build_configured_embedding_provider() - database = memory_database_path(repo_root) + database = memory_workspace(repo_root) with MemoryStore(database, provider) as store: items = store.retrieve( MemoryQuery( @@ -299,7 +291,8 @@ def retrieve_memories( public = [_public_item(item, semantics + " " + diagnostic_text) for item in items] return { "status": "found" if items else "empty", - "database": database.as_posix(), + "database": "mysql", + "workspace": database.as_posix(), "query": { "operator": operator_name, "failure_stage": failure_stage, @@ -312,7 +305,8 @@ def retrieve_memories( except Exception as error: return { "status": "unavailable", - "database": memory_database_path(repo_root).as_posix(), + "database": "mysql", + "workspace": memory_workspace(repo_root).as_posix(), "query": { "operator": operator_name, "failure_stage": failure_stage, diff --git a/packages/dsh-triton-riscv/python/codex_agent/evaluate_memory_modes.py b/packages/dsh-triton-riscv/python/codex_agent/evaluate_memory_modes.py index 777735d..0c9e28e 100644 --- a/packages/dsh-triton-riscv/python/codex_agent/evaluate_memory_modes.py +++ b/packages/dsh-triton-riscv/python/codex_agent/evaluate_memory_modes.py @@ -78,7 +78,7 @@ def evaluate_modes( if limit < 1 or not 0.0 <= lexical_weight <= 1.0: raise ValueError("limit must be positive and lexical_weight must be between 0 and 1") with tempfile.TemporaryDirectory() as directory: - database = Path(directory) / "memory.sqlite3" + database = Path(directory) with MemoryStore(database) as store: sources = set() for item in payload["corpus"]: diff --git a/packages/dsh-triton-riscv/python/codex_agent/evaluate_memory_retrieval.py b/packages/dsh-triton-riscv/python/codex_agent/evaluate_memory_retrieval.py index a7b673e..884685d 100644 --- a/packages/dsh-triton-riscv/python/codex_agent/evaluate_memory_retrieval.py +++ b/packages/dsh-triton-riscv/python/codex_agent/evaluate_memory_retrieval.py @@ -25,7 +25,7 @@ def load_fixture(path: Path) -> dict[str, Any]: def evaluate_fixture(payload: dict[str, Any]) -> dict[str, Any]: with tempfile.TemporaryDirectory() as temporary: - with MemoryStore(Path(temporary) / "memory.sqlite3") as store: + with MemoryStore(Path(temporary)) as store: source_to_id: dict[str, int] = {} for item in payload["corpus"]: memory_id, _ = store.add(MemoryRecord(**item)) diff --git a/packages/dsh-triton-riscv/python/codex_agent/harness/mcp_server.py b/packages/dsh-triton-riscv/python/codex_agent/harness/mcp_server.py index 8d9ecb7..f7f56a1 100644 --- a/packages/dsh-triton-riscv/python/codex_agent/harness/mcp_server.py +++ b/packages/dsh-triton-riscv/python/codex_agent/harness/mcp_server.py @@ -115,15 +115,36 @@ def retrieve_operator_memory_tool( error_text=error_text, limit=limit, candidate_strategy=candidate_strategy) +def _submit_validation_task(root, operator, run_id): + from codex_agent.tasks.service import TaskService + job = TaskService(root).submit_validation(operator, run_id, "validation:" + run_id) + if job["status"] in {"succeeded", "failed"} and job["result"].get("receipt_path"): + return job["result"] + return {"status": "queued" if job["status"] in {"queued", "running"} else "blocked", + "async_job": True, "queue_status": job["status"], "job_id": job["id"], "run_id": run_id, + "message": "Not a test result. End this turn; inspect_queued_task reads persisted progress later."} + + @server.tool(name="execute_approved_validation", structured_output=True, annotations=ToolAnnotations(read_only_hint=False, destructive_hint=True, open_world_hint=True)) -def execute_approved_validation_tool(run_id: str) -> ValidationToolResult: +def execute_approved_validation_tool(run_id: str) -> dict[str, Any]: """Execute the exact host-approved plan. After SSH loss, use the SAME run_id to collect its original remote job, not restart. Unknown evidence needs host inspection. The model cannot grant approval.""" from codex_agent.operator_lifecycle import _load_receipt root = repository_root() receipt = _load_receipt(root, run_id) + from codex_agent.runtime_config import runtime_config + if runtime_config().queue.enabled: + return _submit_validation_task(root, receipt["operator"], run_id) return validate_operator_target(root, receipt["operator"], execute=True, approved_run_id=run_id, - source_env=receipt.get("source_env", True), timeout_seconds=receipt.get("timeout_seconds", 900)) + source_env=receipt.get("source_env", True), timeout_seconds=receipt.get("timeout_seconds", 900)).model_dump() + + +@server.tool(name="inspect_queued_task", structured_output=True, + annotations=ToolAnnotations(read_only_hint=True, destructive_hint=False, open_world_hint=False)) +def inspect_queued_task_tool(job_id: str) -> dict[str, Any]: + """Read persisted queue/worker results; queued or published never means tests passed. No wait loop, replay or execution.""" + from codex_agent.tasks.store import JobStore + return JobStore(repository_root()).detail(job_id) @server.tool(name="get_validation_status", structured_output=True, @@ -322,7 +343,12 @@ def validate_operator_tool( approved_run_id: str | None = None, source_env: bool = True, timeout_seconds: int = 900, -) -> ValidationToolResult: +) -> dict[str, Any]: + from codex_agent.runtime_config import runtime_config + if execute and runtime_config().queue.enabled: + if not approved_run_id: + raise PermissionError("an approved validation plan is required") + return _submit_validation_task(repository_root(), operator_name, approved_run_id) return validate_operator_target( repository_root(), operator_name, @@ -330,7 +356,7 @@ def validate_operator_tool( approved_run_id=approved_run_id, source_env=source_env, timeout_seconds=timeout_seconds, - ) + ).model_dump() @server.tool( diff --git a/packages/dsh-triton-riscv/python/codex_agent/harness/runtime.py b/packages/dsh-triton-riscv/python/codex_agent/harness/runtime.py index d72390d..00b9e18 100644 --- a/packages/dsh-triton-riscv/python/codex_agent/harness/runtime.py +++ b/packages/dsh-triton-riscv/python/codex_agent/harness/runtime.py @@ -81,6 +81,11 @@ def _runtime(self) -> Any: "TRITON_RISCV_REQUIRE_APPROVED_VALIDATION", "1" ), } + from codex_agent.runtime_config import CONFIG_ENV, runtime_config + config = runtime_config() + for key in (CONFIG_ENV, config.storage.urlEnv, config.cache.urlEnv, config.queue.urlEnv): + if os.environ.get(key): + runtime_env[key] = os.environ[key] for name, value in os.environ.items(): if name.startswith(("TRITON_RISCV_MEMORY_", "TRITON_RISCV_EMBEDDING_")) or name in { "TRITON_RISCV_STATE_DIR", "AGENT_EMBEDDING_API_KEY", diff --git a/packages/dsh-triton-riscv/python/codex_agent/memory.py b/packages/dsh-triton-riscv/python/codex_agent/memory.py index ad7d68b..e39c026 100644 --- a/packages/dsh-triton-riscv/python/codex_agent/memory.py +++ b/packages/dsh-triton-riscv/python/codex_agent/memory.py @@ -6,13 +6,17 @@ import argparse import hashlib import json -import os import re -import sqlite3 import time from dataclasses import asdict, dataclass, field from pathlib import Path from typing import Iterable +from sqlalchemy import func, select + +from codex_agent.storage.database import WorkspaceDatabase +from codex_agent.storage.cache import ReadCache +from codex_agent.runtime_config import runtime_config +from codex_agent.storage.schema import memories as memory_table, chunks as chunk_table from .embeddings import EmbeddingProvider, build_embedding_provider, cosine_similarity from .memory_selection import CandidateStrategy, select_candidates @@ -298,124 +302,49 @@ def positive_ranks(candidates: list[dict], score_key: str) -> dict[int, int]: class MemoryStore: - """SQLite-backed memory store with soft archival and optional embeddings.""" + """MySQL-backed memories scoped to one canonical workspace (not a file path).""" - def __init__( - self, - path: Path, - embedding_provider: EmbeddingProvider | None = None, - ) -> None: - self.path = path - self.path.parent.mkdir(parents=True, exist_ok=True) + def __init__(self, workspace: Path, embedding_provider: EmbeddingProvider | None = None): + self.db = WorkspaceDatabase(workspace) + self.db.register() self.embedding_provider = embedding_provider - self.connection = sqlite3.connect(path) - self.connection.row_factory = sqlite3.Row - self._initialize() - - def close(self) -> None: - connection = getattr(self, "connection", None) - if connection is not None: - connection.close() - self.connection = None - - def __del__(self) -> None: - try: - self.close() - except Exception: - pass - - def __enter__(self) -> "MemoryStore": + self.cache = ReadCache(self.db) + + def close(self): + # Connections belong to bounded process-local pools, not this store. + pass + + def __enter__(self): return self - def __exit__(self, *_args: object) -> None: + def __exit__(self, *_args): self.close() - def _initialize(self) -> None: - self.connection.executescript( - """ - CREATE TABLE IF NOT EXISTS metadata ( - key TEXT PRIMARY KEY, - value TEXT NOT NULL - ); - CREATE TABLE IF NOT EXISTS memories ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - fingerprint TEXT NOT NULL UNIQUE, - memory_type TEXT NOT NULL, - operator TEXT NOT NULL, - semantics TEXT NOT NULL, - pytorch_reference TEXT NOT NULL, - tl_ops_json TEXT NOT NULL, - failure_stage TEXT, - error_signature TEXT, - environment_json TEXT NOT NULL, - summary TEXT NOT NULL, - searchable_text TEXT NOT NULL, - outcome TEXT NOT NULL, - evidence_json TEXT NOT NULL, - confidence_grade TEXT NOT NULL, - confidence REAL NOT NULL, - source_run TEXT NOT NULL, - test_sha256 TEXT, - active INTEGER NOT NULL DEFAULT 1, - archive_reason TEXT, - created_at TEXT NOT NULL, - updated_at TEXT NOT NULL, - last_used_at TEXT, - useful_count INTEGER NOT NULL DEFAULT 0, - embedding_json TEXT, - embedding_provider TEXT, - embedding_model TEXT, - embedding_dim INTEGER - ); - CREATE INDEX IF NOT EXISTS idx_memories_active ON memories(active); - CREATE INDEX IF NOT EXISTS idx_memories_operator ON memories(operator); - CREATE INDEX IF NOT EXISTS idx_memories_stage ON memories(failure_stage); - CREATE INDEX IF NOT EXISTS idx_memories_error ON memories(error_signature); - CREATE TABLE IF NOT EXISTS memory_chunks ( - memory_id INTEGER NOT NULL, - kind TEXT NOT NULL, - position INTEGER NOT NULL, - text TEXT NOT NULL, - source_field TEXT NOT NULL DEFAULT '', - embedding_json TEXT, - embedding_provider TEXT, - embedding_model TEXT, - embedding_dim INTEGER, - PRIMARY KEY (memory_id, kind, position) - ); - CREATE INDEX IF NOT EXISTS idx_memory_chunks_memory ON memory_chunks(memory_id); - """ - ) - columns = { - row["name"] for row in self.connection.execute("PRAGMA table_info(memory_chunks)") - } - if "source_field" not in columns: - self.connection.execute( - "ALTER TABLE memory_chunks ADD COLUMN source_field TEXT NOT NULL DEFAULT ''" - ) - previous = self.connection.execute( - "SELECT value FROM metadata WHERE key = 'schema_version'" - ).fetchone() - if previous and int(previous["value"]) < SCHEMA_VERSION: - # Parent records remain untouched. Old vectors refer to a different chunk layout. - self.connection.execute("DELETE FROM memory_chunks") - self.connection.execute( - "UPDATE memories SET embedding_json = NULL, embedding_provider = NULL, " - "embedding_model = NULL, embedding_dim = NULL" - ) - missing = self.connection.execute( - """SELECT * FROM memories WHERE NOT EXISTS - (SELECT 1 FROM memory_chunks WHERE memory_chunks.memory_id = memories.id)""" - ).fetchall() - for row in missing: - self._insert_chunks(row["id"], memory_chunks(self._row_dict(row))) - self.connection.execute( - "INSERT OR REPLACE INTO metadata(key, value) VALUES('schema_version', ?)", - (str(SCHEMA_VERSION),), - ) - self.connection.commit() - - def _embedding_for(self, text: str) -> list[float] | None: + def _stored(self, *, active_only=False, connection=None): + conditions = [memory_table.c.active == 1] if active_only else [] + return self.db.rows(memory_table, *conditions, order=(memory_table.c.id,), connection=connection) + + def chunk_rows(self, memory_id=None, *, connection=None): + conditions = [] if memory_id is None else [chunk_table.c.memory_id == memory_id] + return self.db.rows(chunk_table, *conditions, + order=(chunk_table.c.memory_id, chunk_table.c.kind, chunk_table.c.position), connection=connection) + + def get(self, memory_id): + if type(memory_id) is not int or memory_id <= 0: + raise ValueError("memory_id must be a positive integer") + def load(): + rows = self.db.rows(memory_table, memory_table.c.id == memory_id, memory_table.c.active == 1) + return self._row_dict(rows[0]) if rows else None + return self.cache.get("detail", {"id": memory_id}, load, exact_id=memory_id) + + def warm_cache_ids(self): + def identifiers(): + with self.db.transaction() as conn: + return list(conn.execute(select(memory_table.c.id).where( + *self.db.predicate(memory_table, memory_table.c.active == 1))).scalars()) + return self.cache.warm_bloom(identifiers) + + def _embedding_for(self, text): if self.embedding_provider is None: return None if self.embedding_provider.count_tokens(text) > self.embedding_provider.max_input_tokens: @@ -425,189 +354,122 @@ def _embedding_for(self, text: str) -> list[float] | None: raise RuntimeError("embedding provider returned no vector") return vectors[0] - def _insert_chunks( - self, - memory_id: int, - chunks: list[MemoryChunk], - vectors: list[list[float]] | None = None, - ) -> None: - for index, chunk in enumerate(chunks): - vector = vectors[index] if vectors is not None else None - self.connection.execute( - """INSERT OR REPLACE INTO memory_chunks - (memory_id, kind, position, text, source_field, embedding_json, - embedding_provider, embedding_model, embedding_dim) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""", - ( - memory_id, chunk.kind, chunk.position, chunk.text, - chunk.source_field, - json.dumps(vector) if vector else None, - self.embedding_provider.name if vector else None, - self.embedding_provider.model if vector else None, - len(vector) if vector else None, - ), - ) - - def _chunk_policy(self) -> ChunkPolicy: + def _chunk_policy(self): if self.embedding_provider is None: return ChunkPolicy() - # The model's tokenizer is mandatory for vector indexing. Count the - # title and special tokens as part of every input, not just the body. maximum = self.embedding_provider.max_input_tokens limit = min(180, maximum) return ChunkPolicy(max_tokens=limit, overlap_tokens=min(24, limit // 6)) - def _case_chunks(self, item: dict) -> list[MemoryChunk]: - return memory_chunks( - item, + def _case_chunks(self, item): + return memory_chunks(item, token_count=self.embedding_provider.count_tokens if self.embedding_provider else None, - policy=self._chunk_policy(), - ) + policy=self._chunk_policy()) - def rebuild_chunks(self) -> int: - """Rebuild child chunks from stored parents without changing parent IDs.""" - rows = self.connection.execute("SELECT * FROM memories ORDER BY id").fetchall() + def _insert_chunks(self, conn, memory_id, chunks, vectors=None): + for index, chunk in enumerate(chunks): + vector = vectors[index] if vectors is not None else None + self.db.insert(conn, chunk_table, dict(memory_id=memory_id, kind=chunk.kind, + position=chunk.position, text=chunk.text, source_field=chunk.source_field, + embedding_json=json.dumps(vector) if vector else None, + embedding_provider=self.embedding_provider.name if vector else None, + embedding_model=self.embedding_provider.model if vector else None, + embedding_dim=len(vector) if vector else None)) + + def rebuild_chunks(self): changed = 0 - with self.connection: - for row in rows: - item = self._row_dict(row) - chunks = self._case_chunks(item) - existing = self.connection.execute( - "SELECT kind, position, text, source_field FROM memory_chunks WHERE memory_id = ? ORDER BY kind, position", - (row["id"],), - ).fetchall() - old = [(part["kind"], part["position"], part["text"], part["source_field"]) for part in existing] - new = sorted((part.kind, part.position, part.text, part.source_field) for part in chunks) - if old == new: - continue - changed += 1 - self.connection.execute("DELETE FROM memory_chunks WHERE memory_id = ?", (row["id"],)) - self._insert_chunks(row["id"], chunks) + with self.db.transaction() as conn: + self.db.lock(conn, "memory-write") + for row in self._stored(connection=conn): + chunks = self._case_chunks(self._row_dict(row)) + existing = self.chunk_rows(row["id"], connection=conn) + before = [(c["kind"], c["position"], c["text"], c["source_field"]) for c in existing] + after = sorted((c.kind, c.position, c.text, c.source_field) for c in chunks) + if before != after: + self.db.delete(conn, chunk_table, chunk_table.c.memory_id == row["id"]) + self._insert_chunks(conn, row["id"], chunks) + changed += 1 return changed - def add(self, record: MemoryRecord) -> tuple[int, bool]: + def add(self, record): record = record.normalized() fingerprint = memory_fingerprint(record) now = utc_timestamp() - existing = self.connection.execute( - "SELECT id FROM memories WHERE fingerprint = ?", (fingerprint,) - ).fetchone() - if existing: - old = self._row_dict(self.connection.execute( - "SELECT * FROM memories WHERE id = ?", (existing["id"],) - ).fetchone()) - if any(old[key] != getattr(record, key) for key in ( - "semantics", "pytorch_reference", "summary", "outcome", "evidence", "environment", "source_run", "tl_ops" - )): - refreshed_chunks = self._case_chunks(asdict(record)) - searchable = record.searchable_text() - full_vector = None - chunk_vectors = None - if self.embedding_provider: - full_ok = self.embedding_provider.count_tokens(searchable) <= self.embedding_provider.max_input_tokens - vectors = self.embedding_provider.embed( - ([searchable] if full_ok else []) - + [embedding_input(record.operator, chunk) for chunk in refreshed_chunks] - ) - if len(vectors) != int(full_ok) + len(refreshed_chunks) or any(not vector for vector in vectors): - raise RuntimeError("embedding provider returned invalid vectors") - full_vector = vectors[0] if full_ok else None - chunk_vectors = vectors[int(full_ok):] - self.connection.execute( - """UPDATE memories SET semantics = ?, pytorch_reference = ?, - summary = ?, outcome = ?, evidence_json = ?, searchable_text = ?, - embedding_json = ?, embedding_provider = ?, embedding_model = ?, - embedding_dim = ? - WHERE id = ?""", - (record.semantics, record.pytorch_reference, record.summary, - record.outcome, json.dumps(record.evidence, sort_keys=True), - searchable, json.dumps(full_vector) if full_vector else None, - self.embedding_provider.name if full_vector else None, - self.embedding_provider.model if full_vector else None, - len(full_vector) if full_vector else None, existing["id"]), - ) - self.connection.execute("DELETE FROM memory_chunks WHERE memory_id = ?", (existing["id"],)) - self._insert_chunks(existing["id"], refreshed_chunks, chunk_vectors) - self.connection.execute( - "UPDATE memories SET updated_at = ?, active = 1, archive_reason = NULL, " - "environment_json = ?, source_run = ?, tl_ops_json = ? WHERE id = ?", - (now, json.dumps(record.environment, sort_keys=True), record.source_run, - json.dumps(record.tl_ops), existing["id"]), - ) - self.connection.commit() - return int(existing["id"]), False + refresh = {"active": 1, "archive_reason": None, "updated_at": now} + + def unchanged(row): + old = self._row_dict(row) + return all(old[key] == getattr(record, key) for key in ( + "semantics", "pytorch_reference", "summary", "outcome", "evidence", + "environment", "source_run", "tl_ops", + )) + if self.embedding_provider: + # Check duplicates before spending any embedding calls; no model call holds a lock. + with self.db.transaction() as conn: + self.db.lock(conn, "memory-write") + existing = self.db.rows(memory_table, memory_table.c.fingerprint == fingerprint, + connection=conn, lock=True) + if existing and unchanged(existing[0]): + memory_id = existing[0]["id"] + self.db.update(conn, memory_table, refresh, memory_table.c.id == memory_id) + return memory_id, False searchable = record.searchable_text() chunks = self._case_chunks(asdict(record)) - full_text_embeddable = ( - self.embedding_provider is not None - and self.embedding_provider.count_tokens(searchable) <= self.embedding_provider.max_input_tokens - ) - vectors = ( - self.embedding_provider.embed( - ([searchable] if full_text_embeddable else []) - + [embedding_input(record.operator, chunk) for chunk in chunks] - ) if self.embedding_provider else None - ) - if vectors is not None and ( - len(vectors) != int(full_text_embeddable) + len(chunks) - or any(not vector for vector in vectors) - ): + full_ok = (self.embedding_provider is not None + and self.embedding_provider.count_tokens(searchable) <= self.embedding_provider.max_input_tokens) + vectors = self.embedding_provider.embed( + ([searchable] if full_ok else []) + + [embedding_input(record.operator, chunk) for chunk in chunks] + ) if self.embedding_provider else None + if vectors is not None and (len(vectors) != int(full_ok) + len(chunks) or any(not v for v in vectors)): raise RuntimeError("embedding provider returned invalid vectors") - vector = vectors[0] if vectors and full_text_embeddable else None - provider_name = self.embedding_provider.name if vector else None - provider_model = self.embedding_provider.model if vector else None - cursor = self.connection.execute( - """ - INSERT INTO memories( - fingerprint, memory_type, operator, semantics, pytorch_reference, - tl_ops_json, failure_stage, error_signature, environment_json, - summary, searchable_text, outcome, evidence_json, confidence_grade, - confidence, source_run, test_sha256, created_at, updated_at, - embedding_json, embedding_provider, embedding_model, embedding_dim - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - """, - ( - fingerprint, - record.memory_type, - record.operator, - record.semantics, - record.pytorch_reference, - json.dumps(record.tl_ops, sort_keys=True), - record.failure_stage, - record.error_signature, - json.dumps(record.environment, sort_keys=True), - record.summary, - searchable, - record.outcome, - json.dumps(record.evidence, sort_keys=True), - record.confidence_grade, - GRADE_CONFIDENCE[record.confidence_grade], - record.source_run, - record.test_sha256, - now, - now, - json.dumps(vector) if vector else None, - provider_name, - provider_model, - len(vector) if vector else None, - ), - ) - memory_id = int(cursor.lastrowid) - self._insert_chunks(memory_id, chunks, vectors[int(full_text_embeddable):] if vectors else None) - self.connection.commit() - return memory_id, True - - def archive(self, memory_id: int, reason: str) -> bool: - cursor = self.connection.execute( - "UPDATE memories SET active = 0, archive_reason = ?, updated_at = ? WHERE id = ?", - (normalize_text(reason), utc_timestamp(), memory_id), - ) - self.connection.commit() - return cursor.rowcount > 0 - - def _row_dict(self, row: sqlite3.Row) -> dict: + vector = vectors[0] if vectors and full_ok else None + values = asdict(record) + for source, target in (("tl_ops", "tl_ops_json"), ("environment", "environment_json"), + ("evidence", "evidence_json")): + values[target] = json.dumps(values.pop(source), sort_keys=True) + values.update(fingerprint=fingerprint, searchable_text=searchable, + confidence=GRADE_CONFIDENCE[record.confidence_grade], updated_at=now, + active=1, archive_reason=None, embedding_json=json.dumps(vector) if vector else None, + embedding_provider=self.embedding_provider.name if vector else None, + embedding_model=self.embedding_provider.model if vector else None, + embedding_dim=len(vector) if vector else None) + # Compute embeddings before locking. Parent/chunks become visible atomically. + with self.db.transaction() as conn: + self.db.lock(conn, "memory-write") + existing = self.db.rows(memory_table, memory_table.c.fingerprint == values["fingerprint"], + connection=conn, lock=True) + created = not existing + if existing: + memory_id = existing[0]["id"] + if unchanged(existing[0]): + # Re-ingesting unchanged evidence must not discard its vectors. + self.db.update(conn, memory_table, refresh, memory_table.c.id == memory_id) + return memory_id, False + update_fields = {key: values[key] for key in ( + "semantics", "pytorch_reference", "summary", "outcome", "evidence_json", + "searchable_text", "environment_json", "source_run", "tl_ops_json", + "embedding_json", "embedding_provider", "embedding_model", "embedding_dim", + )} + self.db.update(conn, memory_table, {**update_fields, **refresh}, memory_table.c.id == memory_id) + self.db.delete(conn, chunk_table, chunk_table.c.memory_id == memory_id) + else: + memory_id = self.db.allocate(conn, "memories") + self.db.insert(conn, memory_table, dict(id=memory_id, created_at=now, **values)) + self._insert_chunks(conn, memory_id, chunks, vectors[int(full_ok):] if vectors else None) + return memory_id, created + + def archive(self, memory_id, reason): + with self.db.transaction() as conn: + return self.db.update(conn, memory_table, + {"active": 0, "archive_reason": normalize_text(reason), "updated_at": utc_timestamp()}, + memory_table.c.id == memory_id).rowcount > 0 + + @staticmethod + def _row_dict(row: dict) -> dict: item = dict(row) + item.pop("workspace_id", None) for source, destination in ( ("tl_ops_json", "tl_ops"), ("environment_json", "environment"), @@ -619,12 +481,8 @@ def _row_dict(self, row: sqlite3.Row) -> dict: item["active"] = bool(item["active"]) return item - def list(self, *, active_only: bool = True) -> list[dict]: - where = "WHERE active = 1" if active_only else "" - rows = self.connection.execute( - f"SELECT * FROM memories {where} ORDER BY id" - ).fetchall() - return [self._row_dict(row) for row in rows] + def list(self, *, active_only=True): + return [self._row_dict(row) for row in self._stored(active_only=active_only)] def retrieve( self, @@ -639,18 +497,26 @@ def retrieve( ) -> list[dict]: if limit <= 0: return [] - candidates = self.rank_candidates( - query, exclude_source_runs=exclude_source_runs, - score_mode=score_mode, lexical_weight=lexical_weight, - ) - selected = select_candidates(candidates, limit, strategy=candidate_strategy, trace=selection_trace) + excluded = sorted(set(exclude_source_runs)) + parameters = {"query": asdict(query), "limit": limit, "excluded": excluded, + "mode": score_mode, "weight": lexical_weight, "strategy": candidate_strategy, + "embedding": runtime_config().memory.embedding.model_dump(), + "provider": [getattr(self.embedding_provider, key, None) + for key in ("name", "model", "max_input_tokens")]} + def load(): + trace = {} + candidates = self.rank_candidates(query, exclude_source_runs=excluded, + score_mode=score_mode, lexical_weight=lexical_weight) + items = select_candidates(candidates, limit, strategy=candidate_strategy, trace=trace) + return {"items": items, "trace": trace} + result = self.cache.get("retrieval", parameters, load) + selected = result["items"] + if selection_trace is not None: + selection_trace.update(result["trace"]) now = utc_timestamp() - for item in selected: - self.connection.execute( - "UPDATE memories SET last_used_at = ? WHERE id = ?", - (now, item["id"]), - ) - self.connection.commit() + with self.db.transaction() as conn: + for item in selected: + self.db.update(conn, memory_table, {"last_used_at": now}, memory_table.c.id == item["id"]) return selected def rank_candidates( @@ -668,15 +534,13 @@ def rank_candidates( raise ValueError("lexical_weight must be between 0 and 1") if score_mode in {"embedding", "fusion"} and self.embedding_provider is None: raise RuntimeError(f"{score_mode} retrieval requires an embedding provider") - rows = self.connection.execute( - "SELECT * FROM memories WHERE active = 1 ORDER BY id" - ).fetchall() + rows = self._stored(active_only=True) query_tokens = tokens(query.searchable_text()) query_ops = set(query.tl_ops) query_stage = canonical_stage(query.failure_stage) query_vector = self._embedding_for(query.searchable_text()) if score_mode == "legacy" else None query_vectors: list[tuple[str, list[float]]] = [] - chunks_by_memory: dict[int, list[sqlite3.Row]] = {} + chunks_by_memory: dict[int, list[dict]] = {} if score_mode in {"embedding", "fusion"}: policy = self._chunk_policy() sections = query_embedding_sections( @@ -688,11 +552,8 @@ def rank_candidates( if len(section_vectors) != len(sections) or any(not vector for vector in section_vectors): raise RuntimeError("embedding provider returned invalid query vectors") query_vectors = [(kind, vector) for (kind, _), vector in zip(sections, section_vectors)] - chunk_rows = self.connection.execute( - """SELECT memory_chunks.* FROM memory_chunks - JOIN memories ON memories.id = memory_chunks.memory_id - WHERE memories.active = 1 ORDER BY memory_id, kind, position""" - ).fetchall() + active_ids = {row["id"] for row in rows} + chunk_rows = [chunk for chunk in self.chunk_rows() if chunk["memory_id"] in active_ids] for chunk in chunk_rows: chunks_by_memory.setdefault(chunk["memory_id"], []).append(chunk) excluded = {normalize_text(item) for item in exclude_source_runs if item} @@ -809,12 +670,11 @@ def rank_candidates( return [item for _score, item in sorted(scored, key=lambda pair: (-pair[0], pair[1]["id"]))] - def mark_useful(self, memory_id: int) -> None: - self.connection.execute( - "UPDATE memories SET useful_count = useful_count + 1, updated_at = ? WHERE id = ?", - (utc_timestamp(), memory_id), - ) - self.connection.commit() + def mark_useful(self, memory_id): + with self.db.transaction() as conn: + self.db.update(conn, memory_table, + {"useful_count": memory_table.c.useful_count + 1, "updated_at": utc_timestamp()}, + memory_table.c.id == memory_id) def maintain(self, low_confidence_days: int = 90) -> dict: """Soft-archive stale low-confidence and superseded diagnostic records.""" @@ -869,109 +729,63 @@ def maintain(self, low_confidence_days: int = 90) -> dict: ) return {"archived": len(archived), "items": archived} - def embed_missing(self, batch_size: int = 32) -> dict: + def embed_missing(self, batch_size=32): if self.embedding_provider is None: raise RuntimeError("embedding provider is not configured") if batch_size < 1: raise ValueError("batch_size must be positive") - self._chunk_policy() self.rebuild_chunks() - rows = self.connection.execute( - """SELECT id, searchable_text FROM memories WHERE active = 1 - AND (embedding_json IS NULL OR embedding_provider IS NOT ? - OR embedding_model IS NOT ?) ORDER BY id""", - (self.embedding_provider.name, self.embedding_provider.model), - ).fetchall() - rows = [row for row in rows if self.embedding_provider.count_tokens(row["searchable_text"]) <= self.embedding_provider.max_input_tokens] - chunk_rows = self.connection.execute( - """SELECT memory_chunks.memory_id, memory_chunks.kind, - memory_chunks.position, memory_chunks.text, - memory_chunks.source_field, memories.operator - FROM memory_chunks JOIN memories ON memories.id = memory_chunks.memory_id - WHERE memories.active = 1 AND - (memory_chunks.embedding_json IS NULL OR - memory_chunks.embedding_provider IS NOT ? OR - memory_chunks.embedding_model IS NOT ?) - ORDER BY memory_id, kind, position""", - (self.embedding_provider.name, self.embedding_provider.model), - ).fetchall() - for offset in range(0, len(rows), batch_size): - batch = rows[offset:offset + batch_size] - vectors = self.embedding_provider.embed([row["searchable_text"] for row in batch]) - if len(vectors) != len(batch) or any(not vector for vector in vectors): - raise RuntimeError("embedding provider returned invalid vectors") - for row, vector in zip(batch, vectors): - self.connection.execute( - """ - UPDATE memories SET embedding_json = ?, embedding_provider = ?, - embedding_model = ?, embedding_dim = ? WHERE id = ? - """, - ( - json.dumps(vector), - self.embedding_provider.name, - self.embedding_provider.model, - len(vector), - row["id"], - ), - ) - self.connection.commit() - for offset in range(0, len(chunk_rows), batch_size): - batch = chunk_rows[offset:offset + batch_size] - inputs = [ - chunk_title(row["operator"], row["kind"], row["source_field"]) + row["text"] - for row in batch - ] - if any(self.embedding_provider.count_tokens(text) > self._chunk_policy().max_tokens for text in inputs): - raise RuntimeError("chunk exceeds embedding token budget after title") - vectors = self.embedding_provider.embed(inputs) - if len(vectors) != len(batch) or any(not vector for vector in vectors): - raise RuntimeError("embedding provider returned invalid chunk vectors") - for row, vector in zip(batch, vectors): - self.connection.execute( - """UPDATE memory_chunks SET embedding_json = ?, - embedding_provider = ?, embedding_model = ?, embedding_dim = ? - WHERE memory_id = ? AND kind = ? AND position = ?""", - ( - json.dumps(vector), self.embedding_provider.name, - self.embedding_provider.model, len(vector), - row["memory_id"], row["kind"], row["position"], - ), - ) - self.connection.commit() - return { - "embedded": len(rows), - "embedded_chunks": len(chunk_rows), - "provider": self.embedding_provider.name, - "model": self.embedding_provider.model, - } - - def stats(self) -> dict: - rows = self.connection.execute( - "SELECT memory_type, active, COUNT(*) AS count FROM memories GROUP BY memory_type, active" - ).fetchall() - total = self.connection.execute("SELECT COUNT(*) FROM memories").fetchone()[0] - embedded = self.connection.execute( - "SELECT COUNT(*) FROM memories WHERE embedding_json IS NOT NULL" - ).fetchone()[0] - embedded_chunks = self.connection.execute( - "SELECT COUNT(*) FROM memory_chunks WHERE embedding_json IS NOT NULL" - ).fetchone()[0] - return { - "schema_version": SCHEMA_VERSION, - "total": total, - "active": sum(row["count"] for row in rows if row["active"]), - "archived": sum(row["count"] for row in rows if not row["active"]), - "embedded": embedded, - "embedded_chunks": embedded_chunks, - "by_type": { - row["memory_type"]: sum( - candidate["count"] - for candidate in rows - if candidate["memory_type"] == row["memory_type"] and candidate["active"] - ) - for row in rows - }, - } + provider = self.embedding_provider + def missing(row): + return (row["embedding_json"] is None or row["embedding_provider"] != provider.name + or row["embedding_model"] != provider.model) + active = self._stored(active_only=True) + parents = {row["id"]: row for row in active} + rows = [row for row in active if missing(row) + and provider.count_tokens(row["searchable_text"]) <= provider.max_input_tokens] + chunks = [row for row in self.chunk_rows() if row["memory_id"] in parents and missing(row)] + for group, table, is_chunk in ((rows, memory_table, False), (chunks, chunk_table, True)): + for offset in range(0, len(group), batch_size): + batch = group[offset:offset + batch_size] + inputs = [chunk_title(parents[row["memory_id"]]["operator"], row["kind"], row["source_field"]) + row["text"] + if is_chunk else row["searchable_text"] for row in batch] + if is_chunk and any(provider.count_tokens(t) > self._chunk_policy().max_tokens for t in inputs): + raise RuntimeError("chunk exceeds embedding token budget after title") + vectors = provider.embed(inputs) + if len(vectors) != len(batch) or any(not vector for vector in vectors): + raise RuntimeError("embedding provider returned invalid vectors") + with self.db.transaction() as conn: + self.db.lock(conn, "memory-write") + for row, vector in zip(batch, vectors): + conditions = ([table.c.memory_id == row["memory_id"], table.c.kind == row["kind"], + table.c.position == row["position"], table.c.text == row["text"], + table.c.source_field == row["source_field"]] + if is_chunk else [table.c.id == row["id"], + table.c.searchable_text == row["searchable_text"]]) + self.db.update(conn, table, dict(embedding_json=json.dumps(vector), + embedding_provider=provider.name, embedding_model=provider.model, + embedding_dim=len(vector)), *conditions) + return {"embedded": len(rows), "embedded_chunks": len(chunks), + "provider": provider.name, "model": provider.model} + + def stats(self): + return self.cache.get("stats", {}, self._stats) + + def _stats(self): + with self.db.transaction() as conn: + rows = list(conn.execute(select(memory_table.c.memory_type, memory_table.c.active, + func.count().label("count")).where(*self.db.predicate(memory_table)).group_by( + memory_table.c.memory_type, memory_table.c.active)).mappings()) + embedded = conn.execute(select(func.count()).select_from(memory_table).where( + *self.db.predicate(memory_table, memory_table.c.embedding_json.is_not(None)))).scalar_one() + embedded_chunks = conn.execute(select(func.count()).select_from(chunk_table).where( + *self.db.predicate(chunk_table, chunk_table.c.embedding_json.is_not(None)))).scalar_one() + return {"schema_version": SCHEMA_VERSION, "total": sum(r["count"] for r in rows), + "active": sum(r["count"] for r in rows if r["active"]), + "archived": sum(r["count"] for r in rows if not r["active"]), + "embedded": embedded, "embedded_chunks": embedded_chunks, + "by_type": {kind: sum(r["count"] for r in rows if r["memory_type"] == kind and r["active"]) + for kind in {r["memory_type"] for r in rows}}} def environment_summary(environment: dict, final: dict) -> dict: @@ -1288,7 +1102,7 @@ def build_provider_from_args(args: argparse.Namespace) -> EmbeddingProvider | No def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description="Manage Triton-RISCV agent memory.") - parser.add_argument("--db", default="agent-results/memory.sqlite3") + parser.add_argument("--workspace", type=Path, default=Path.cwd(), help="Target workspace in MySQL") subparsers = parser.add_subparsers(dest="command", required=True) ingest = subparsers.add_parser("ingest", help="Import completed development runs.") @@ -1310,6 +1124,11 @@ def parse_args() -> argparse.Namespace: stats_parser = subparsers.add_parser("stats", help="Print memory statistics.") add_embedding_arguments(stats_parser) + detail = subparsers.add_parser("show", help="Read one active historical case.") + detail.add_argument("memory_id", type=int) + add_embedding_arguments(detail) + warm = subparsers.add_parser("warm-cache-ids", help="Build an optional revision-scoped Bloom filter.") + add_embedding_arguments(warm) archive = subparsers.add_parser("archive", help="Soft-archive one memory.") archive.add_argument("memory_id", type=int) @@ -1332,7 +1151,7 @@ def parse_args() -> argparse.Namespace: def main() -> int: args = parse_args() provider = build_provider_from_args(args) - with MemoryStore(Path(args.db), provider) as store: + with MemoryStore(args.workspace, provider) as store: if args.command == "ingest": result = ingest_results(store, Path(args.results_dir)) elif args.command == "search": @@ -1352,6 +1171,10 @@ def main() -> int: ) elif args.command == "archive": result = {"archived": store.archive(args.memory_id, args.reason)} + elif args.command == "show": + result = store.get(args.memory_id) + elif args.command == "warm-cache-ids": + result = store.warm_cache_ids() elif args.command == "embed-missing": result = store.embed_missing() elif args.command == "rebuild-chunks": diff --git a/packages/dsh-triton-riscv/python/codex_agent/memory_api.py b/packages/dsh-triton-riscv/python/codex_agent/memory_api.py index fa040aa..cfa8a1d 100644 --- a/packages/dsh-triton-riscv/python/codex_agent/memory_api.py +++ b/packages/dsh-triton-riscv/python/codex_agent/memory_api.py @@ -43,6 +43,7 @@ class RetrievedMemoryCase(BaseModel): class MemoryRetrievalToolResult(BaseModel): status: Literal["found", "empty", "unavailable"] database: str + workspace: str | None = None query: dict[str, Any] items: list[RetrievedMemoryCase] = Field(default_factory=list) warning: str | None = None diff --git a/packages/dsh-triton-riscv/python/codex_agent/messaging/__init__.py b/packages/dsh-triton-riscv/python/codex_agent/messaging/__init__.py new file mode 100644 index 0000000..7e6ee4e --- /dev/null +++ b/packages/dsh-triton-riscv/python/codex_agent/messaging/__init__.py @@ -0,0 +1 @@ +"""RabbitMQ transport. Task state remains in MySQL.""" diff --git a/packages/dsh-triton-riscv/python/codex_agent/messaging/broker.py b/packages/dsh-triton-riscv/python/codex_agent/messaging/broker.py new file mode 100644 index 0000000..d18a1df --- /dev/null +++ b/packages/dsh-triton-riscv/python/codex_agent/messaging/broker.py @@ -0,0 +1,53 @@ +"""One workspace-scoped durable queue per worker role, with confirmed publishing.""" +import json +import os +from urllib.parse import urlsplit + +import aio_pika +from pamqp.commands import Basic + +from codex_agent.runtime_config import runtime_config + + +class Broker: + def __init__(self, jobs): + self.jobs = jobs + self.prefix = f"triton.{jobs.db.cache_namespace}.{jobs.db.workspace_id}" + self.connection = None + + async def __aenter__(self): + cfg = runtime_config().queue + url = os.environ.get(cfg.urlEnv, "") + if not cfg.enabled or urlsplit(url).scheme not in {"amqp", "amqps"}: + raise ValueError(f"Enable queue and set {cfg.urlEnv} to an AMQP URL") + try: + self.connection = await aio_pika.connect(url, timeout=5) + self.channel = await self.connection.channel(publisher_confirms=True, on_return_raises=True) + await self.channel.set_qos(prefetch_count=1) + self.exchange = await self.channel.declare_exchange(self.prefix, aio_pika.ExchangeType.DIRECT, durable=True) + dead = await self.channel.declare_exchange(self.prefix + ".dead", aio_pika.ExchangeType.FANOUT, durable=True) + queue = await self.channel.declare_queue(self.prefix + ".dead", durable=True) + await queue.bind(dead) + self.queues = {} + for kind in ("agent", "validation", "memory"): + queue = await self.channel.declare_queue(self.prefix + "." + kind, durable=True, + arguments={"x-dead-letter-exchange": dead.name}) + await queue.bind(self.exchange, routing_key=kind) + self.queues[kind] = queue + return self + except Exception: + if self.connection is not None: + await self.connection.close() + raise RuntimeError("RabbitMQ unavailable or topology incompatible; check service/configuration") from None + + async def __aexit__(self, *_): + if self.connection is not None: + await self.connection.close() + + async def publish(self, event): + message = aio_pika.Message(json.dumps(self.jobs.envelope(event)).encode(), + content_type="application/json", delivery_mode=aio_pika.DeliveryMode.PERSISTENT, + message_id=event["id"], type=event["kind"]) + confirmed = await self.exchange.publish(message, routing_key=event["kind"], mandatory=True, timeout=5) + if not isinstance(confirmed, Basic.Ack): + raise RuntimeError("RabbitMQ did not confirm publication") diff --git a/packages/dsh-triton-riscv/python/codex_agent/messaging/relay.py b/packages/dsh-triton-riscv/python/codex_agent/messaging/relay.py new file mode 100644 index 0000000..bd20b9c --- /dev/null +++ b/packages/dsh-triton-riscv/python/codex_agent/messaging/relay.py @@ -0,0 +1,16 @@ +"""Polling transactional outbox. Network waits never hold a database transaction.""" +import asyncio + + +async def publish_once(jobs, broker): + event = await asyncio.to_thread(jobs.claim_publication) + if event is None: + return False + try: + await broker.publish(event) + except Exception as error: + # The exception message can contain a credential-bearing AMQP URL. + await asyncio.to_thread(jobs.publication_result, event, type(error).__name__) + else: + await asyncio.to_thread(jobs.publication_result, event) + return True diff --git a/packages/dsh-triton-riscv/python/codex_agent/migrate_memory.py b/packages/dsh-triton-riscv/python/codex_agent/migrate_memory.py deleted file mode 100644 index 948e787..0000000 --- a/packages/dsh-triton-riscv/python/codex_agent/migrate_memory.py +++ /dev/null @@ -1,124 +0,0 @@ -"""Copy a historical database and referenced evidence without opening the original MemoryStore.""" -from __future__ import annotations - -import argparse -import hashlib -import json -from pathlib import Path -import shutil -import sqlite3 - -from codex_agent.memory import MemoryStore - -ALLOWED_SUFFIXES = {".json", ".jsonl", ".log", ".txt", ".md", ".diff", ".patch", ".py", ".mlir", ".ll"} -PATH_KEYS = {"source", "source_run", "path", "log_path", "receipt_path", "patch_path", "final_result_path", "task_file"} - - -def migrate_memory(source: Path, source_root: Path, destination: Path) -> dict: - source, source_root, destination = source.resolve(), source_root.resolve(), destination.resolve() - if destination.exists(): - raise FileExistsError("destination must be new; no history will be overwritten") - if not source.is_file(): - raise FileNotFoundError(source) - source.relative_to(source_root) - destination.mkdir(parents=True, mode=0o700) - database = destination / "memory.sqlite3" - original_hash = hashlib.sha256(source.read_bytes()).hexdigest() - manifest = {"source_database": str(source), "source_root": str(source_root), - "source_database_sha256_before": original_hash, "records": [], "files": [], "gaps": [], - "note": "Historical labels preserved, not newly audited. Missing evidence is not a verified result."} - # SQLite backup includes committed WAL state; plain file copying may lose it. - with sqlite3.connect(source.as_uri() + "?mode=ro", uri=True) as original: - with sqlite3.connect(database) as copied: - original.backup(copied) - database.chmod(0o600) - copied_files = {} - total_bytes = 0 - - def archive(value: str) -> str: - nonlocal total_bytes - candidate = Path(value) - if not candidate.is_absolute(): - candidate = source_root / candidate - candidate = candidate.resolve() - if not candidate.is_relative_to(source_root): - manifest["gaps"].append({"source": value, "reason": "outside approved source root"}) - return value - if not candidate.exists(): - if value.startswith(("agent-results/", "/")): - manifest["gaps"].append({"source": value, "reason": "missing"}) - return value - paths = sorted(candidate.rglob("*")) if candidate.is_dir() else [candidate] - for item in paths: - if not item.is_file() or item.suffix not in ALLOWED_SUFFIXES or item.is_symlink(): - continue - resolved = item.resolve() - if not resolved.is_relative_to(source_root) or any(p.startswith(".") for p in item.relative_to(source_root).parts): - continue - if str(item) in copied_files: - continue - size = item.stat().st_size - if size > 16 * 1024**2 or total_bytes + size > 100 * 1024**2: - manifest["gaps"].append({"source": str(item), "reason": "archive size budget exceeded"}) - continue - target = destination / "evidence" / item.relative_to(source_root) - target.parent.mkdir(parents=True, exist_ok=True) - shutil.copyfile(item, target) - target.chmod(0o600) - total_bytes += size - entry = {"source": str(item), "copy": str(target), "bytes": size, - "sha256": hashlib.sha256(target.read_bytes()).hexdigest()} - manifest["files"].append(entry) - copied_files[str(item)] = str(target) - if candidate.is_dir() and (destination / "evidence" / candidate.relative_to(source_root)).exists(): - return str(destination / "evidence" / candidate.relative_to(source_root)) - return copied_files.get(str(candidate), value) - - def relocate(value, key=""): - if isinstance(value, dict): - return {k: relocate(v, k) for k, v in value.items()} - if isinstance(value, list): - return [relocate(v, key) for v in value] - if isinstance(value, str) and key in PATH_KEYS and value: - return archive(value) - return value - - # Only the independent copy may undergo schema upgrades or evidence relocation. - with MemoryStore(database) as store: - rows = store.connection.execute("SELECT id, source_run, evidence_json FROM memories").fetchall() - for row in rows: - evidence = relocate(json.loads(row["evidence_json"])) - original_run = row["source_run"] - new_run = archive(original_run) if original_run.startswith(("agent-results/", "/")) else original_run - manifest["records"].append({"memory_id": row["id"], "original_source_run": original_run, "source_run": new_run}) - store.connection.execute("UPDATE memories SET source_run=?, evidence_json=? WHERE id=?", - (new_run, json.dumps(evidence, ensure_ascii=False), row["id"])) - # Rebuild only this copy's chunks after changing provenance; old vectors are stale. - store.connection.execute("DELETE FROM memory_chunks") - store.connection.execute("UPDATE memories SET embedding_json=NULL, embedding_provider=NULL, embedding_model=NULL, embedding_dim=NULL") - store.connection.commit() - with MemoryStore(database) as store: - manifest["stats"] = store.stats() - manifest["source_database_sha256_after"] = hashlib.sha256(source.read_bytes()).hexdigest() - manifest["source_file_unchanged"] = manifest["source_database_sha256_after"] == original_hash - manifest["database"] = str(database) - manifest["archived_bytes"] = total_bytes - (destination / "migration-manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2), encoding="utf-8") - return manifest - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--source", required=True, type=Path) - parser.add_argument("--source-root", required=True, type=Path) - parser.add_argument("--destination", required=True, type=Path) - args = parser.parse_args() - result = migrate_memory(args.source, args.source_root, args.destination) - print(json.dumps({"database": result["database"], "records": len(result["records"]), - "archived_files": len(result["files"]), "gaps": len(result["gaps"]), - "source_file_unchanged": result["source_file_unchanged"]}, indent=2)) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/packages/dsh-triton-riscv/python/codex_agent/operator_lifecycle.py b/packages/dsh-triton-riscv/python/codex_agent/operator_lifecycle.py index 53f42a7..2d13bf2 100644 --- a/packages/dsh-triton-riscv/python/codex_agent/operator_lifecycle.py +++ b/packages/dsh-triton-riscv/python/codex_agent/operator_lifecycle.py @@ -21,6 +21,7 @@ from codex_agent.operator_tools import discover_operator_evidence from codex_agent.diagnostic_memory import remember_validation +from codex_agent.tasks.context import defer_memory from codex_agent.memory_api import MemoryRetrievalToolResult, RetrievedMemoryCase, retrieve_operator_memory from codex_agent.remote_executor import ( RemotePreflightResult, @@ -398,7 +399,8 @@ def validate_operator_target( ) # Only executed, audited outcomes become historical evidence. payload["memory_write"] = ( - remember_validation(root, payload, operator) + ({"status": "deferred", "reason": "validation worker will enqueue audited evidence"} + if defer_memory.get() else remember_validation(root, payload, operator)) if execute and result.failure_stage not in {"capacity", "cancellation"} and validation_evidence.verdict in {"verified-passed", "verified-failed"} else {"status": "not-recorded", "reason": "no verified execution"} ) diff --git a/packages/dsh-triton-riscv/python/codex_agent/platform/api.py b/packages/dsh-triton-riscv/python/codex_agent/platform/api.py index f9a1650..0c3ba4c 100644 --- a/packages/dsh-triton-riscv/python/codex_agent/platform/api.py +++ b/packages/dsh-triton-riscv/python/codex_agent/platform/api.py @@ -9,6 +9,7 @@ from typing import AsyncIterator from fastapi import FastAPI, HTTPException, Query +from fastapi.responses import Response from fastapi.responses import FileResponse, JSONResponse, StreamingResponse from fastapi.staticfiles import StaticFiles from pydantic import BaseModel, Field @@ -28,6 +29,9 @@ from codex_agent.platform.executor import HarnessRunExecutor from codex_agent.platform.service import PlatformService from codex_agent.platform.store import PlatformStore +from codex_agent.runtime_config import runtime_config +from codex_agent.tasks.service import TaskService +from codex_agent.tasks.store import Conflict class SessionRequest(BaseModel): @@ -40,6 +44,13 @@ class SessionUpdateRequest(BaseModel): class MessageRequest(BaseModel): content: str = Field(min_length=1, max_length=12000) + request_id: str | None = Field(default=None, min_length=1, max_length=160) + + +class ValidationJobRequest(BaseModel): + operator: str = Field(min_length=1, max_length=255) + approved_run_id: str = Field(min_length=1, max_length=120) + request_id: str = Field(min_length=1, max_length=160) class RepairDecisionRequest(BaseModel): @@ -55,11 +66,13 @@ def create_app( root = (repo_root or repository_root()).resolve() frontend_dist = Path(__file__).parents[1] / "frontend" / "dist" react_frontend_available = (frontend_dist / "index.html").is_file() - store = PlatformStore(state_root(root) / "platform.sqlite3") + store = PlatformStore(root) settings = harness_agent.settings if harness_agent else HarnessSettings.from_env(root) agent = harness_agent or HarnessAgent(settings) service = PlatformService(root, store, agent) - executor = HarnessRunExecutor(root, store, agent) + queued = runtime_config().queue.enabled + tasks = TaskService(root) if queued else None + executor = HarnessRunExecutor(root, store, agent, workers=0 if queued else 2) @asynccontextmanager async def lifespan(_app: FastAPI) -> AsyncIterator[None]: @@ -76,6 +89,7 @@ async def lifespan(_app: FastAPI) -> AsyncIterator[None]: app.state.store = store app.state.service = service app.state.executor = executor + app.state.tasks = tasks if react_frontend_available: app.mount( "/ui-assets", @@ -100,6 +114,7 @@ def health() -> dict: return { "status": "ok", "repo_root": root.as_posix(), + "execution_mode": "rabbitmq" if queued else "inline", "harness": { "provider": settings.provider, "model": settings.model, @@ -153,11 +168,19 @@ def session_approvals(session_id: str) -> list[dict]: raise HTTPException(status_code=404, detail=str(error)) from error @app.post("/api/sessions/{session_id}/messages") - def message(session_id: str, request: MessageRequest) -> dict: + def message(session_id: str, request: MessageRequest, response: Response) -> dict: try: + if tasks is not None: + if not request.request_id: + raise HTTPException(status_code=400, detail="request_id is required in queue mode; reuse it when retrying") + result = tasks.submit_turn(service, session_id, request.content, request.request_id) + response.status_code = 202 + return result result = service.handle_message(session_id, request.content) result["run"] = executor.schedule(result["run"]["id"]) return result + except Conflict as error: + raise HTTPException(status_code=409, detail=str(error)) from error except KeyError as error: raise HTTPException(status_code=404, detail=str(error)) from error except ValueError as error: @@ -173,12 +196,54 @@ def run(run_id: str) -> dict: @app.post("/api/runs/{run_id}/confirm") def confirm(run_id: str) -> dict: try: + if queued: + return store.get_run(run_id) return executor.confirm(run_id) except KeyError as error: raise HTTPException(status_code=404, detail=str(error)) from error except ValueError as error: raise HTTPException(status_code=400, detail=str(error)) from error + def task_service(): + if tasks is None: + raise HTTPException(status_code=409, detail="queue execution is disabled") + return tasks + + @app.post("/api/jobs/validation", status_code=202) + def submit_validation_job(request: ValidationJobRequest): + try: + return task_service().submit_validation(request.operator, request.approved_run_id, request.request_id) + except PermissionError as error: + raise HTTPException(status_code=403, detail=str(error)) from error + except KeyError as error: + raise HTTPException(status_code=404, detail=str(error)) from error + except ValueError as error: + raise HTTPException(status_code=409, detail=str(error)) from error + + @app.get("/api/jobs/{job_id}") + def job(job_id: str): + try: + return task_service().jobs.detail(job_id) + except KeyError as error: + raise HTTPException(status_code=404, detail="unknown job") from error + + @app.post("/api/jobs/{job_id}/cancel") + def cancel_job(job_id: str): + try: + return task_service().jobs.cancel(job_id) + except KeyError as error: + raise HTTPException(status_code=404, detail="unknown job") from error + + @app.get("/api/jobs/{job_id}/artifacts/{artifact_id}") + def job_artifact(job_id: str, artifact_id: str): + from codex_agent.artifacts import ArtifactStore + detail = job(job_id) + record = next((a for a in detail["artifacts"] if a["id"] == artifact_id), None) + if record is None: + raise HTTPException(status_code=404, detail="unknown artifact") + return Response(ArtifactStore(root).read(record), media_type="application/octet-stream", + headers={"Content-Disposition": 'attachment; filename="' + record["name"] + '"'}) + @app.get("/api/runs/{run_id}/events") def events(run_id: str, after: int = Query(default=-1, ge=-1)) -> list[dict]: try: diff --git a/packages/dsh-triton-riscv/python/codex_agent/platform/executor.py b/packages/dsh-triton-riscv/python/codex_agent/platform/executor.py index a7809b5..42cf379 100644 --- a/packages/dsh-triton-riscv/python/codex_agent/platform/executor.py +++ b/packages/dsh-triton-riscv/python/codex_agent/platform/executor.py @@ -37,7 +37,7 @@ def __init__( self.repo_root = repo_root.resolve() self.store = store self.agent = agent - self.pool = ThreadPoolExecutor(max_workers=workers, thread_name_prefix="dsh-agent") + self.pool = ThreadPoolExecutor(max_workers=workers, thread_name_prefix="dsh-agent") if workers else None self._active: set[str] = set() self._lock = threading.Lock() settings = agent.settings @@ -77,6 +77,8 @@ def schedule(self, run_id: str) -> dict: return self.store.get_run(run_id) def _queue(self, run_id: str, *, phase: str) -> None: + if self.pool is None: + raise RuntimeError("This executor is worker-owned; submit through the task service") self.store.update_run(run_id, status="queued", phase=phase) with self._lock: if run_id not in self._active: @@ -84,7 +86,8 @@ def _queue(self, run_id: str, *, phase: str) -> None: self.pool.submit(self._execute, run_id) def close(self) -> None: - self.pool.shutdown(wait=True, cancel_futures=True) + if self.pool is not None: + self.pool.shutdown(wait=True, cancel_futures=True) self.agent.close() @staticmethod diff --git a/packages/dsh-triton-riscv/python/codex_agent/platform/requirements.txt b/packages/dsh-triton-riscv/python/codex_agent/platform/requirements.txt index dd1b7b5..5a74b7c 100644 --- a/packages/dsh-triton-riscv/python/codex_agent/platform/requirements.txt +++ b/packages/dsh-triton-riscv/python/codex_agent/platform/requirements.txt @@ -1,3 +1,8 @@ fastapi>=0.115 httpx>=0.28 uvicorn[standard]>=0.34 +SQLAlchemy>=2.0,<2.1 +PyMySQL[rsa]>=1.1,<2 +alembic>=1.13,<2 +redis>=5,<7 +aio-pika>=9.5,<10 diff --git a/packages/dsh-triton-riscv/python/codex_agent/platform/store.py b/packages/dsh-triton-riscv/python/codex_agent/platform/store.py index 85e970f..a584e72 100644 --- a/packages/dsh-triton-riscv/python/codex_agent/platform/store.py +++ b/packages/dsh-triton-riscv/python/codex_agent/platform/store.py @@ -1,15 +1,15 @@ -"""SQLite persistence for chat sessions, messages, runs, and event streams.""" - +"""MySQL persistence for workbench conversations, runs and event streams.""" from __future__ import annotations import json -import sqlite3 -import threading import time import uuid -from contextlib import contextmanager from pathlib import Path -from typing import Iterator + +from sqlalchemy import select +from sqlalchemy.dialects.mysql import insert +from codex_agent.storage.database import WorkspaceDatabase +from codex_agent.storage.schema import sessions, messages, runs, events, checkpoints def timestamp() -> str: @@ -17,374 +17,159 @@ def timestamp() -> str: class PlatformStore: - def __init__(self, path: Path) -> None: - self.path = path - self.path.parent.mkdir(parents=True, exist_ok=True) - self._lock = threading.RLock() - self._initialize() - - def _connect(self) -> sqlite3.Connection: - connection = sqlite3.connect(self.path, timeout=30) - connection.row_factory = sqlite3.Row - connection.execute("PRAGMA journal_mode=WAL") - connection.execute("PRAGMA foreign_keys=ON") - return connection - - @contextmanager - def _connection(self) -> Iterator[sqlite3.Connection]: - connection = self._connect() - try: - yield connection - connection.commit() - except Exception: - connection.rollback() - raise - finally: - connection.close() - - def _initialize(self) -> None: - with self._connection() as connection: - connection.executescript( - """ - CREATE TABLE IF NOT EXISTS sessions ( - id TEXT PRIMARY KEY, - title TEXT NOT NULL, - status TEXT NOT NULL, - created_at TEXT NOT NULL, - updated_at TEXT NOT NULL, - pinned INTEGER NOT NULL DEFAULT 0 - ); - CREATE TABLE IF NOT EXISTS messages ( - id TEXT PRIMARY KEY, - session_id TEXT NOT NULL REFERENCES sessions(id) ON DELETE CASCADE, - role TEXT NOT NULL, - content TEXT NOT NULL, - metadata_json TEXT NOT NULL, - created_at TEXT NOT NULL - ); - CREATE INDEX IF NOT EXISTS idx_messages_session ON messages(session_id, created_at); - CREATE TABLE IF NOT EXISTS runs ( - id TEXT PRIMARY KEY, - session_id TEXT NOT NULL REFERENCES sessions(id) ON DELETE CASCADE, - intent TEXT NOT NULL, - operator TEXT, - status TEXT NOT NULL, - phase TEXT NOT NULL, - request_json TEXT NOT NULL, - result_json TEXT NOT NULL, - created_at TEXT NOT NULL, - updated_at TEXT NOT NULL - ); - CREATE INDEX IF NOT EXISTS idx_runs_session ON runs(session_id, created_at); - CREATE TABLE IF NOT EXISTS events ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - run_id TEXT NOT NULL REFERENCES runs(id) ON DELETE CASCADE, - sequence INTEGER NOT NULL, - event_type TEXT NOT NULL, - payload_json TEXT NOT NULL, - created_at TEXT NOT NULL, - UNIQUE(run_id, sequence) - ); - CREATE INDEX IF NOT EXISTS idx_events_run ON events(run_id, sequence); - CREATE TABLE IF NOT EXISTS context_checkpoints ( - session_id TEXT PRIMARY KEY REFERENCES sessions(id) ON DELETE CASCADE, - summary TEXT NOT NULL, - through_message_id TEXT, - source_ids_json TEXT NOT NULL, - metrics_json TEXT NOT NULL, - updated_at TEXT NOT NULL - ); - """ - ) - session_columns = { - row["name"] - for row in connection.execute("PRAGMA table_info(sessions)").fetchall() - } - if "pinned" not in session_columns: - connection.execute( - "ALTER TABLE sessions ADD COLUMN pinned INTEGER NOT NULL DEFAULT 0" - ) + def __init__(self, workspace: Path) -> None: + self.db = WorkspaceDatabase(workspace) + self.db.register() @staticmethod - def _row(row: sqlite3.Row | None) -> dict | None: + def _row(row): if row is None: return None item = dict(row) - for key in ("metadata_json", "request_json", "result_json", "payload_json", "source_ids_json", "metrics_json"): + for key in ("workspace_id", "ordinal", "next_sequence"): + item.pop(key, None) + for key in ("metadata_json", "request_json", "result_json", "payload_json", + "source_ids_json", "metrics_json"): if key in item: item[key.removesuffix("_json")] = json.loads(item.pop(key) or "{}") return item - def create_session(self, title: str = "新对话") -> dict: - session_id = uuid.uuid4().hex - now = timestamp() - with self._lock, self._connection() as connection: - connection.execute( - """ - INSERT INTO sessions (id, title, status, created_at, updated_at, pinned) - VALUES (?, ?, 'active', ?, ?, 0) - """, - (session_id, title[:80] or "新对话", now, now), - ) - return self.get_session(session_id) - - def get_session(self, session_id: str) -> dict: - with self._connection() as connection: - row = connection.execute( - "SELECT * FROM sessions WHERE id = ?", (session_id,) - ).fetchone() - item = self._row(row) - if item is None: - raise KeyError(f"unknown session: {session_id}") - return item - - def list_sessions(self, limit: int = 50) -> list[dict]: - with self._connection() as connection: - rows = connection.execute( - "SELECT * FROM sessions ORDER BY pinned DESC, updated_at DESC LIMIT ?", - (limit,), - ).fetchall() - return [self._row(row) for row in rows] - - def set_session_pinned(self, session_id: str, pinned: bool) -> dict: - self.get_session(session_id) - with self._lock, self._connection() as connection: - connection.execute( - "UPDATE sessions SET pinned = ? WHERE id = ?", - (int(pinned), session_id), - ) + def _get(self, table, identifier, *, connection=None, lock=False): + rows = self.db.rows(table, table.c.id == identifier, connection=connection, lock=lock) + if not rows: + raise KeyError(f"unknown {table.name}: {identifier}") + return rows[0] + + def create_session(self, title="新对话"): + identifier, now = uuid.uuid4().hex, timestamp() + with self.db.transaction() as conn: + self.db.insert(conn, sessions, dict(id=identifier, title=title[:80] or "新对话", + status="active", created_at=now, updated_at=now, pinned=0)) + return self.get_session(identifier) + + def get_session(self, session_id): + return self._row(self._get(sessions, session_id)) + + def list_sessions(self, limit=50): + return [self._row(row) for row in self.db.rows(sessions, + order=(sessions.c.pinned.desc(), sessions.c.updated_at.desc(), sessions.c.id), limit=limit)] + + def set_session_pinned(self, session_id, pinned): + with self.db.transaction() as conn: + self._get(sessions, session_id, connection=conn, lock=True) + self.db.update(conn, sessions, {"pinned": int(pinned)}, sessions.c.id == session_id) return self.get_session(session_id) - def delete_session(self, session_id: str) -> None: - self.get_session(session_id) - with self._lock, self._connection() as connection: - active_run = connection.execute( - """ - SELECT 1 FROM runs - WHERE session_id = ? AND status IN ('queued', 'running') - LIMIT 1 - """, - (session_id,), - ).fetchone() - if active_run is not None: + def delete_session(self, session_id): + with self.db.transaction() as conn: + self._get(sessions, session_id, connection=conn, lock=True) + active = self.db.rows(runs, runs.c.session_id == session_id, + runs.c.status.in_(("queued", "running")), connection=conn, limit=1) + if active: raise ValueError("cannot delete a session while a run is active") - connection.execute("DELETE FROM sessions WHERE id = ?", (session_id,)) - - def list_session_events( - self, - session_id: str, - *, - event_type: str | None = None, - ) -> list[dict]: - self.get_session(session_id) - query = """ - SELECT events.* - FROM events - JOIN runs ON runs.id = events.run_id - WHERE runs.session_id = ? - """ - parameters: list[object] = [session_id] - if event_type is not None: - query += " AND events.event_type = ?" - parameters.append(event_type) - query += " ORDER BY events.created_at, events.id" - with self._connection() as connection: - rows = connection.execute(query, parameters).fetchall() - return [self._row(row) for row in rows] - - def update_session_title(self, session_id: str, title: str) -> None: - with self._lock, self._connection() as connection: - connection.execute( - "UPDATE sessions SET title = ?, updated_at = ? WHERE id = ?", - (title[:80], timestamp(), session_id), - ) - - def add_message( - self, - session_id: str, - role: str, - content: str, - metadata: dict | None = None, - ) -> dict: - message_id = uuid.uuid4().hex - now = timestamp() - with self._lock, self._connection() as connection: - connection.execute( - "INSERT INTO messages VALUES (?, ?, ?, ?, ?, ?)", - ( - message_id, - session_id, - role, - content, - json.dumps(metadata or {}, sort_keys=True), - now, - ), - ) - connection.execute( - "UPDATE sessions SET updated_at = ? WHERE id = ?", (now, session_id) - ) - return self.get_message(message_id) - - def get_message(self, message_id: str) -> dict: - with self._connection() as connection: - row = connection.execute( - "SELECT * FROM messages WHERE id = ?", (message_id,) - ).fetchone() - item = self._row(row) - if item is None: - raise KeyError(f"unknown message: {message_id}") - return item - - def list_messages(self, session_id: str) -> list[dict]: - with self._connection() as connection: - rows = connection.execute( - "SELECT * FROM messages WHERE session_id = ? ORDER BY created_at, rowid", - (session_id,), - ).fetchall() - return [self._row(row) for row in rows] - - def create_run( - self, - session_id: str, - intent: str, - operator: str | None, - request: dict, - *, - status: str = "awaiting-confirmation", - phase: str = "planned", - ) -> dict: - run_id = uuid.uuid4().hex - now = timestamp() - with self._lock, self._connection() as connection: - connection.execute( - "INSERT INTO runs VALUES (?, ?, ?, ?, ?, ?, ?, '{}', ?, ?)", - ( - run_id, - session_id, - intent, - operator, - status, - phase, - json.dumps(request, sort_keys=True), - now, - now, - ), - ) - self.add_event(run_id, "planned", {"intent": intent, "operator": operator}) - return self.get_run(run_id) - - def get_run(self, run_id: str) -> dict: - with self._connection() as connection: - row = connection.execute( - "SELECT * FROM runs WHERE id = ?", (run_id,) - ).fetchone() - item = self._row(row) - if item is None: - raise KeyError(f"unknown run: {run_id}") - return item - - def list_runs(self, session_id: str) -> list[dict]: - with self._connection() as connection: - rows = connection.execute( - "SELECT * FROM runs WHERE session_id = ? ORDER BY created_at DESC", - (session_id,), - ).fetchall() - return [self._row(row) for row in rows] - - def update_run( - self, - run_id: str, - *, - status: str | None = None, - phase: str | None = None, - result: dict | None = None, - ) -> dict: - current = self.get_run(run_id) - with self._lock, self._connection() as connection: - connection.execute( - "UPDATE runs SET status = ?, phase = ?, result_json = ?, updated_at = ? WHERE id = ?", - ( - status or current["status"], - phase or current["phase"], - json.dumps(result if result is not None else current["result"], sort_keys=True), - timestamp(), - run_id, - ), - ) + from codex_agent.tasks.schema import jobs + run_ids = select(runs.c.id).where(*self.db.predicate(runs, runs.c.session_id == session_id)) + if self.db.rows(jobs, jobs.c.run_id.in_(run_ids), jobs.c.status == "needs-reconciliation", connection=conn, limit=1): + raise ValueError("resolve the uncertain task before deleting its session") + self.db.delete(conn, sessions, sessions.c.id == session_id) + + def update_session_title(self, session_id, title): + with self.db.transaction() as conn: + self._get(sessions, session_id, connection=conn, lock=True) + self.db.update(conn, sessions, {"title": title[:80], "updated_at": timestamp()}, + sessions.c.id == session_id) + + def add_message(self, session_id, role, content, metadata=None): + identifier, now = uuid.uuid4().hex, timestamp() + with self.db.transaction() as conn: + self._get(sessions, session_id, connection=conn, lock=True) + self.db.insert(conn, messages, dict(id=identifier, session_id=session_id, + ordinal=self.db.allocate(conn, "messages"), role=role, content=content, + metadata_json=json.dumps(metadata or {}, sort_keys=True), created_at=now)) + self.db.update(conn, sessions, {"updated_at": now}, sessions.c.id == session_id) + return self.get_message(identifier) + + def get_message(self, message_id): + return self._row(self._get(messages, message_id)) + + def list_messages(self, session_id): + return [self._row(row) for row in self.db.rows(messages, messages.c.session_id == session_id, + order=(messages.c.ordinal,))] + + def create_run(self, session_id, intent, operator, request, *, + status="awaiting-confirmation", phase="planned"): + identifier, now = uuid.uuid4().hex, timestamp() + with self.db.transaction() as conn: + self._get(sessions, session_id, connection=conn, lock=True) + self.db.insert(conn, runs, dict(id=identifier, session_id=session_id, intent=intent, + operator=operator, status=status, phase=phase, + request_json=json.dumps(request, sort_keys=True), result_json="{}", + created_at=now, updated_at=now, next_sequence=0)) + self._add_event(conn, identifier, "planned", {"intent": intent, "operator": operator}) + return self.get_run(identifier) + + def get_run(self, run_id): + return self._row(self._get(runs, run_id)) + + def list_runs(self, session_id): + return [self._row(row) for row in self.db.rows(runs, runs.c.session_id == session_id, + order=(runs.c.created_at.desc(), runs.c.id))] + + def update_run(self, run_id, *, status=None, phase=None, result=None): + with self.db.transaction() as conn: + current = self._get(runs, run_id, connection=conn) + # Lock the session first, matching delete_session/create_run lock order. + self._get(sessions, current["session_id"], connection=conn, lock=True) + current = self._get(runs, run_id, connection=conn, lock=True) + values = {"status": status or current["status"], "phase": phase or current["phase"], + "updated_at": timestamp()} + if result is not None: + values["result_json"] = json.dumps(result, sort_keys=True) + self.db.update(conn, runs, values, runs.c.id == run_id) return self.get_run(run_id) - def add_event(self, run_id: str, event_type: str, payload: dict) -> dict: - with self._lock, self._connection() as connection: - sequence = connection.execute( - "SELECT COALESCE(MAX(sequence), -1) + 1 FROM events WHERE run_id = ?", - (run_id,), - ).fetchone()[0] - cursor = connection.execute( - "INSERT INTO events(run_id, sequence, event_type, payload_json, created_at) VALUES (?, ?, ?, ?, ?)", - (run_id, sequence, event_type, json.dumps(payload, sort_keys=True), timestamp()), - ) - row = connection.execute( - "SELECT * FROM events WHERE id = ?", (cursor.lastrowid,) - ).fetchone() - return self._row(row) - - def list_events(self, run_id: str, after: int = -1) -> list[dict]: - with self._connection() as connection: - rows = connection.execute( - "SELECT * FROM events WHERE run_id = ? AND sequence > ? ORDER BY sequence", - (run_id, after), - ).fetchall() - return [self._row(row) for row in rows] - - def session_bundle(self, session_id: str) -> dict: - return { - "session": self.get_session(session_id), - "messages": self.list_messages(session_id), - "runs": self.list_runs(session_id), - } - - def get_context_checkpoint(self, session_id: str) -> dict | None: - with self._connection() as connection: - row = connection.execute( - "SELECT * FROM context_checkpoints WHERE session_id = ?", (session_id,) - ).fetchone() - return self._row(row) - - - def save_context_checkpoint( - self, - session_id: str, - *, - summary: str, - through_message_id: str | None, - source_ids: list[str], - metrics: dict, - ) -> dict: + def _add_event(self, conn, run_id, event_type, payload): + run = self._get(runs, run_id, connection=conn, lock=True) + sequence = run["next_sequence"] + self.db.update(conn, runs, {"next_sequence": sequence + 1}, runs.c.id == run_id) + value = dict(id=self.db.allocate(conn, "events"), run_id=run_id, sequence=sequence, + event_type=event_type, payload_json=json.dumps(payload, sort_keys=True), + created_at=timestamp()) + self.db.insert(conn, events, value) + return self._row(value) + + def add_event(self, run_id, event_type, payload): + with self.db.transaction() as conn: + return self._add_event(conn, run_id, event_type, payload) + + def list_events(self, run_id, after=-1): + return [self._row(row) for row in self.db.rows(events, events.c.run_id == run_id, + events.c.sequence > after, order=(events.c.sequence,))] + + def list_session_events(self, session_id, *, event_type=None): self.get_session(session_id) - now = timestamp() - with self._lock, self._connection() as connection: - connection.execute( - """ - INSERT INTO context_checkpoints( - session_id, summary, through_message_id, source_ids_json, - metrics_json, updated_at - ) VALUES (?, ?, ?, ?, ?, ?) - ON CONFLICT(session_id) DO UPDATE SET - summary = excluded.summary, - through_message_id = excluded.through_message_id, - source_ids_json = excluded.source_ids_json, - metrics_json = excluded.metrics_json, - updated_at = excluded.updated_at - """, - ( - session_id, - summary, - through_message_id, - json.dumps(source_ids, sort_keys=True), - json.dumps(metrics, sort_keys=True), - now, - ), - ) - checkpoint = self.get_context_checkpoint(session_id) - assert checkpoint is not None - return checkpoint + selected_runs = select(runs.c.id).where(*self.db.predicate(runs, runs.c.session_id == session_id)) + conditions = [events.c.run_id.in_(selected_runs)] + if event_type is not None: + conditions.append(events.c.event_type == event_type) + return [self._row(row) for row in self.db.rows(events, *conditions, + order=(events.c.created_at, events.c.id))] + + def session_bundle(self, session_id): + return {"session": self.get_session(session_id), "messages": self.list_messages(session_id), + "runs": self.list_runs(session_id)} + + def get_context_checkpoint(self, session_id): + rows = self.db.rows(checkpoints, checkpoints.c.session_id == session_id) + return self._row(rows[0]) if rows else None + + def save_context_checkpoint(self, session_id, *, summary, through_message_id, source_ids, metrics): + with self.db.transaction() as conn: + self._get(sessions, session_id, connection=conn, lock=True) + values = dict(workspace_id=self.db.workspace_id, session_id=session_id, summary=summary, + through_message_id=through_message_id, source_ids_json=json.dumps(source_ids, sort_keys=True), + metrics_json=json.dumps(metrics, sort_keys=True), updated_at=timestamp()) + stmt = insert(checkpoints).values(**values) + conn.execute(stmt.on_duplicate_key_update(**{ + key: stmt.inserted[key] for key in values if key not in {"workspace_id", "session_id"}})) + return self.get_context_checkpoint(session_id) diff --git a/packages/dsh-triton-riscv/python/codex_agent/process_control.py b/packages/dsh-triton-riscv/python/codex_agent/process_control.py index fcd1e6e..56c3fb3 100644 --- a/packages/dsh-triton-riscv/python/codex_agent/process_control.py +++ b/packages/dsh-triton-riscv/python/codex_agent/process_control.py @@ -19,11 +19,17 @@ class ExecutionCancelled(RuntimeError): def validation_environment() -> dict[str, str]: """Do not hand model credentials or plugin approval switches to test code.""" from codex_agent.runtime_config import CONFIG_ENV, runtime_config - credential = runtime_config().memory.embedding.apiKeyEnv + settings = runtime_config() + credential = settings.memory.embedding.apiKeyEnv + database_credential = settings.storage.urlEnv + cache_credential = settings.cache.urlEnv + queue_credential = settings.queue.urlEnv return {key: value for key, value in os.environ.items() if not key.startswith("TRITON_RISCV_ALLOW_") and not key.endswith(("API_KEY", "ACCESS_TOKEN", "AUTH_TOKEN", "PASSWORD")) - and key not in {CONFIG_ENV, credential, "SSH_AUTH_SOCK", "GH_TOKEN", "GITHUB_TOKEN"}} + and key not in {CONFIG_ENV, credential, database_credential, cache_credential, queue_credential, + "TRITON_AMQP_URL", + "TRITON_REDIS_URL", "TRITON_MYSQL_URL", "SSH_AUTH_SOCK", "GH_TOKEN", "GITHUB_TOKEN"}} @contextmanager diff --git a/packages/dsh-triton-riscv/python/codex_agent/reference_library.py b/packages/dsh-triton-riscv/python/codex_agent/reference_library.py index 8e852f2..b348262 100644 --- a/packages/dsh-triton-riscv/python/codex_agent/reference_library.py +++ b/packages/dsh-triton-riscv/python/codex_agent/reference_library.py @@ -15,7 +15,9 @@ from pathlib import Path import re import shlex -import sqlite3 +from codex_agent.storage.database import WorkspaceDatabase +from codex_agent.storage.schema import references as catalog_table +from codex_agent.storage.cache import ReadCache import subprocess from codex_agent.memory import SECRET_PATTERNS, jaccard, tokens @@ -324,11 +326,13 @@ def build_library(root: Path, output: Path, *, excluded_operators: set[str] | No seen.add(record["reference_id"]) records.append(record) output.mkdir(parents=True) - database = output / "references.sqlite3" - with sqlite3.connect(database) as connection: - connection.execute("CREATE TABLE references_catalog (id TEXT PRIMARY KEY, admitted INTEGER NOT NULL, payload TEXT NOT NULL)") - connection.executemany("INSERT INTO references_catalog VALUES (?, ?, ?)", - [(r["reference_id"], r["decision"] == "admitted", json.dumps(r, ensure_ascii=False)) for r in records]) + database = WorkspaceDatabase(output) + database.register() + with database.transaction() as connection: + for record in records: + database.insert(connection, catalog_table, { + "id": record["reference_id"], "admitted": int(record["decision"] == "admitted"), + "payload": json.dumps(record, ensure_ascii=False)}) for decision, filename in (("admitted", "admitted.jsonl"), ("quarantined", "quarantine.jsonl")): with (output / filename).open("x", encoding="utf-8") as stream: for record in records: @@ -342,7 +346,7 @@ def build_library(root: Path, output: Path, *, excluded_operators: set[str] | No "quarantined": sum(r["decision"] == "quarantined" for r in records), "quarantine_reasons": dict(sorted(reasons.items())), "rejected_receipts": rejected_receipts, "excluded_operators": sorted(excluded), - "product_database_modified": False, "model_or_remote_calls": 0, + "product_memory_modified": False, "catalog_storage": "mysql", "model_or_remote_calls": 0, "limitations": ["heuristic static checks are not a sandbox or proof of correctness", "no arbitrary docs or PR claims promoted to verified rules", "historical validation may not match current source hashes", @@ -358,16 +362,12 @@ def search_library(library: Path, query: str, *, environment: dict, raise ValueError("limit must be between 1 and 10") if environment.get("architecture") != "riscv64" or not environment.get("triton"): raise ValueError("explicit riscv64 architecture and Triton version required") - database = library.resolve() / "references.sqlite3" - connection = sqlite3.connect(database.as_uri() + "?mode=ro", uri=True) - try: - connection.execute("PRAGMA query_only=ON") - rows = connection.execute("SELECT payload FROM references_catalog WHERE admitted=1 ORDER BY id").fetchall() - finally: - connection.close() + database = WorkspaceDatabase(library) + rows = ReadCache(database, "references").get("admitted", {}, lambda: + database.rows(catalog_table, catalog_table.c.admitted == 1, order=(catalog_table.c.id,))) items, rejected = [], [] - for (payload,) in rows: - record = json.loads(payload) + for row in rows: + record = json.loads(row["payload"]) if (record.get("schema") != SCHEMA or record.get("decision") != "admitted" or record.get("provenance") != "real" or not record.get("validation")): rejected.append({"id": record.get("reference_id"), "reason": "invalid-admission-metadata"}) diff --git a/packages/dsh-triton-riscv/python/codex_agent/runtime_config.py b/packages/dsh-triton-riscv/python/codex_agent/runtime_config.py index b4a0578..27d7218 100644 --- a/packages/dsh-triton-riscv/python/codex_agent/runtime_config.py +++ b/packages/dsh-triton-riscv/python/codex_agent/runtime_config.py @@ -44,13 +44,44 @@ class Embedding(Settings): apiKeyEnv: str = "AGENT_EMBEDDING_API_KEY" +class Storage(Settings): + urlEnv: str = "TRITON_MYSQL_URL" + poolSize: int = Field(default=5, ge=1, le=50) + maxOverflow: int = Field(default=5, ge=0, le=50) + poolTimeout: int = Field(default=10, ge=1, le=60) + + +class Cache(Settings): + enabled: bool = False + urlEnv: str = "TRITON_REDIS_URL" + ttlSeconds: int = Field(default=60, ge=1, le=3600) + negativeTtlSeconds: int = Field(default=20, ge=1, le=60) + lockMs: int = Field(default=10000, ge=100, le=60000) + waitMs: int = Field(default=1000, ge=10, le=5000) + maxBytes: int = Field(default=262144, ge=1024, le=1048576) + maxConcurrent: int = Field(default=4, ge=1, le=32) + requestsPerSecond: int = Field(default=100, ge=1, le=10000) + rebuildsPerSecond: int = Field(default=10, ge=1, le=1000) + bloom: bool = False + bloomCapacity: int = Field(default=10000, ge=100, le=10000000) + bloomErrorRate: float = Field(default=0.001, gt=0, le=0.1) + + class Memory(Settings): - database: str | None = None retrievalMode: str = "legacy" contextFormat: str = "classic" embedding: Embedding = Field(default_factory=Embedding) +class Queue(Settings): + enabled: bool = False + urlEnv: str = "TRITON_AMQP_URL" + leaseSeconds: int = Field(default=60, ge=15, le=600) + maxAttempts: int = Field(default=3, ge=1, le=10) + jobTimeoutSeconds: int = Field(default=1200, ge=30, le=7200) + maxPending: int = Field(default=100, ge=1, le=10000) + + class RuntimeConfig(Settings): schemaVersion: Literal[1] = 1 repoRoot: str = "" @@ -58,6 +89,9 @@ class RuntimeConfig(Settings): permissions: Permissions = Field(default_factory=Permissions) remote: Remote = Field(default_factory=Remote) memory: Memory = Field(default_factory=Memory) + storage: Storage = Field(default_factory=Storage) + cache: Cache = Field(default_factory=Cache) + queue: Queue = Field(default_factory=Queue) def _unique(pairs): @@ -92,7 +126,7 @@ def _parse_native(raw: str) -> RuntimeConfig: for path in (cfg.repoRoot, cfg.stateDir): if not path or not Path(path).is_absolute(): raise ValueError("workspace and state paths must be absolute") - for path in (cfg.memory.database, cfg.memory.embedding.tokenizerJson): + for path in (cfg.memory.embedding.tokenizerJson,): if path and not Path(path).is_absolute(): raise ValueError("memory paths must be absolute") remote = cfg.remote @@ -105,7 +139,22 @@ def _parse_native(raw: str) -> RuntimeConfig: raise ValueError("invalid remote repository") if remote.requireTaskQuotas and not remote.required: raise ValueError("task quotas require remote mode") + db_key = cfg.storage.urlEnv + if (not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", db_key) + or db_key.startswith(("TRITON_RISCV_", "RISCV_")) + or db_key in {"PATH", "HOME", "PYTHONPATH", "PYTHONHOME"}): + raise ValueError("reserved or invalid database credential variable") + cache_key = cfg.cache.urlEnv + if (not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", cache_key) + or cache_key.startswith(("TRITON_RISCV_", "RISCV_")) + or cache_key in {"PATH", "HOME", "PYTHONPATH", "PYTHONHOME", db_key}): + raise ValueError("reserved or invalid cache credential variable") key = cfg.memory.embedding.apiKeyEnv + mq_key = cfg.queue.urlEnv + if (not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", mq_key) + or mq_key.startswith(("TRITON_RISCV_", "RISCV_")) + or mq_key in {"PATH", "HOME", "PYTHONPATH", "PYTHONHOME", db_key, cache_key, key}): + raise ValueError("reserved or invalid queue credential variable") if (not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", key) or key.startswith(("TRITON_RISCV_", "RISCV_")) or key in {"PATH", "HOME", "PYTHONPATH", "PYTHONHOME"}): @@ -141,7 +190,6 @@ def _legacy_config(env: Mapping[str, str]) -> RuntimeConfig: required=env.get("TRITON_RISCV_REQUIRE_REMOTE") == "1", requireTaskQuotas=env.get("TRITON_RISCV_REQUIRE_TASK_QUOTAS", "").strip().lower() in {"1", "true"}), memory=Memory( - database=env.get("TRITON_RISCV_MEMORY_DB"), retrievalMode=env.get("TRITON_RISCV_MEMORY_RETRIEVAL_MODE", "legacy"), contextFormat=env.get("TRITON_RISCV_MEMORY_CONTEXT_FORMAT", "classic"), embedding=Embedding( diff --git a/packages/dsh-triton-riscv/python/codex_agent/storage/__init__.py b/packages/dsh-triton-riscv/python/codex_agent/storage/__init__.py new file mode 100644 index 0000000..fb2b967 --- /dev/null +++ b/packages/dsh-triton-riscv/python/codex_agent/storage/__init__.py @@ -0,0 +1 @@ +"""MySQL persistence shared by the workbench and domain tools.""" diff --git a/packages/dsh-triton-riscv/python/codex_agent/storage/__main__.py b/packages/dsh-triton-riscv/python/codex_agent/storage/__main__.py new file mode 100644 index 0000000..5264240 --- /dev/null +++ b/packages/dsh-triton-riscv/python/codex_agent/storage/__main__.py @@ -0,0 +1,30 @@ +"""Explicit schema upgrades; runtime connections never create or migrate tables.""" +import argparse +from pathlib import Path + +from alembic import command +from alembic.config import Config +from codex_agent.storage.database import engine + + +def upgrade(): + config = Config() + config.set_main_option("script_location", str(Path(__file__).parent / "migrations")) + with engine().connect() as connection: + config.attributes["connection"] = connection + command.upgrade(config, "head") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("command", choices=["upgrade"]) + parser.parse_args() + try: + upgrade() + except Exception: + raise SystemExit("MySQL schema upgrade failed; check connection and migration configuration") from None + print("MySQL schema is up to date") + + +if __name__ == "__main__": + main() diff --git a/packages/dsh-triton-riscv/python/codex_agent/storage/cache.py b/packages/dsh-triton-riscv/python/codex_agent/storage/cache.py new file mode 100644 index 0000000..cc1fe4d --- /dev/null +++ b/packages/dsh-triton-riscv/python/codex_agent/storage/cache.py @@ -0,0 +1,294 @@ +"""Optional read-through Redis cache. MySQL remains the version authority.""" +from collections import Counter +from contextlib import contextmanager +from functools import lru_cache +import hashlib +import json +import logging +import math +import os +import random +import secrets +import threading +import time +from urllib.parse import urlsplit + +import redis +from redis.backoff import NoBackoff +from redis.retry import Retry + +from codex_agent.runtime_config import runtime_config + + +class CacheBusy(RuntimeError): + pass + + +class CacheUnavailable(RuntimeError): + pass + + +class VersionChanged(RuntimeError): + pass + + +RELEASE = """ +if redis.call('GET', KEYS[1]) == ARGV[1] then + return redis.call('DEL', KEYS[1]) +end +return 0 +""" +PUBLISH = """ +if redis.call('GET', KEYS[1]) ~= ARGV[1] then return 0 end +redis.call('SET', KEYS[2], ARGV[2], 'PX', ARGV[3]) +redis.call('DEL', KEYS[1]) +return 1 +""" +RATE = """ +local now = tonumber(redis.call('TIME')[1]) +for i = 1, #KEYS do + local entry = redis.call('HMGET', KEYS[i], 'second', 'count') + if tonumber(entry[1]) == now and tonumber(entry[2]) >= tonumber(ARGV[i]) then return 0 end +end +for i = 1, #KEYS do + if tonumber(redis.call('HGET', KEYS[i], 'second')) ~= now then + redis.call('HSET', KEYS[i], 'second', now, 'count', 0) + end + redis.call('HINCRBY', KEYS[i], 'count', 1) + redis.call('EXPIRE', KEYS[i], 2) +end +return 1 +""" +BLOOM_CHECK = """ +if redis.call('EXISTS', KEYS[1]) == 0 or redis.call('GET', KEYS[2]) ~= 'ready' then return -1 end +return redis.call('BF.EXISTS', KEYS[1], ARGV[1]) +""" +BLOOM_CREATE = """ +redis.call('BF.RESERVE', KEYS[1], ARGV[1], ARGV[2], 'NONSCALING') +redis.call('EXPIRE', KEYS[1], 120) +return 1 +""" + + +def digest(value): + return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(',', ':'), + ensure_ascii=False, allow_nan=False).encode()).hexdigest() + + +class RedisBackend: + def __init__(self, url): + if urlsplit(url).scheme not in {"redis", "rediss", "unix"}: + raise ValueError("Cache URL must use redis, rediss or unix") + try: + self.client = redis.Redis.from_url(url, socket_timeout=0.2, + socket_connect_timeout=0.2, max_connections=32, + retry=Retry(NoBackoff(), 0), decode_responses=False) + except (ValueError, TypeError): + raise ValueError("Invalid cache connection URL") from None + self.lock = threading.Lock() + self.metrics = Counter() + self.retry_at = 0.0 + self.policy_until = 0.0 + self.fallback_slots = threading.BoundedSemaphore(2) + self.fallback_times = [] + + def count(self, name): + with self.lock: + self.metrics[name] += 1 + + def snapshot(self): + with self.lock: + return dict(self.metrics) + + def call(self, function, *args, **kwargs): + if time.monotonic() < self.retry_at: + raise CacheUnavailable("Redis temporarily unavailable") + try: + if time.monotonic() >= self.policy_until: + settings = self.client.config_get('maxmemory*') + policy = settings.get('maxmemory-policy') + if policy not in {b'noeviction', 'noeviction'} or int(settings.get('maxmemory', 0)) <= 0: + raise redis.RedisError("coordination keys must not be evicted") + self.policy_until = time.monotonic() + 5 + return function(*args, **kwargs) + except redis.RedisError: + self.retry_at = time.monotonic() + 2 + self.count("redis_errors") + logging.getLogger(__name__).warning( + "Redis read cache unavailable; bounded MySQL fallback enabled for two seconds") + raise CacheUnavailable("Redis unavailable; require bounded memory and noeviction") from None + + @contextmanager + def fallback(self): + with self.lock: + now = time.monotonic() + self.fallback_times = [t for t in self.fallback_times if t > now - 1] + allowed = len(self.fallback_times) < 4 + if allowed: + self.fallback_times.append(now) + if not allowed or not self.fallback_slots.acquire(blocking=False): + self.count("fallback_rejected") + raise CacheBusy("Historical retrieval is busy while Redis recovers; retry later") + try: + self.count("fallback") + yield + finally: + self.fallback_slots.release() + + +@lru_cache(maxsize=8) +def _backend(url, pid): + return RedisBackend(url) + + +class ReadCache: + def __init__(self, database, domain="memory"): + self.db = database + self.domain = domain + self.config = runtime_config().cache + self.backend = None + self.prefix = f"triton:cache:v1:{database.cache_namespace}:{database.workspace_id}:{domain}" + self.global_prefix = f"triton:cache:v1:{database.cache_namespace}" + if self.config.enabled: + raw = os.environ.get(self.config.urlEnv, "") + if not raw: + raise ValueError(f"Set {self.config.urlEnv} before enabling Redis caching") + self.backend = _backend(raw, os.getpid()) + + def _rate(self, operation, limit): + keys = [self.prefix + ':rate:' + operation, self.global_prefix + ':rate:' + operation] + if not self.backend.call(self.backend.client.eval, RATE, 2, *keys, limit, limit * 4): + self.backend.count("rate_rejected") + raise CacheBusy("Historical retrieval rate limit reached; retry later") + + def key(self, kind, parameters, revision): + return f"{self.prefix}:{revision}:{kind}:{digest(parameters)}" + + def _compute(self, loader, revision): + with self.db.cache_read_slot(self.config.maxConcurrent): + self.backend.count("rebuilds") + result = loader() + if self.db.revision(self.domain) != revision: + raise VersionChanged() + return result + + def get(self, kind, parameters, loader, *, exact_id=None): + if self.backend is None: + return loader() + backend, cfg = self.backend, self.config + backend.count("requests") + try: + self._rate('request', cfg.requestsPerSecond) + for _ in range(2): + revision = self.db.revision(self.domain) + try: + return self._get(kind, parameters, loader, revision, exact_id) + except VersionChanged: + backend.count("version_retries") + raise CacheBusy("History changed during retrieval; retry with the new revision") + except CacheUnavailable: + # No unrestricted fail-open. DB named slots also bound independent processes. + with backend.fallback(): + for _ in range(2): + revision = self.db.revision(self.domain) + try: + return self._compute(loader, revision) + except VersionChanged: + backend.count("version_retries") + raise CacheBusy("History changed during retrieval; retry later") + + def _get(self, kind, parameters, loader, revision, exact_id): + backend, client, cfg = self.backend, self.backend.client, self.config + key = self.key(kind, parameters, revision) + owner = secrets.token_hex(24) + lock_key = key + ':lock' + end = time.monotonic() + cfg.waitMs / 1000 + while True: + cached = backend.call(client.get, key) + if cached is not None: + try: + envelope = json.loads(cached) + if not isinstance(envelope, dict) or set(envelope) != {'value'}: + raise ValueError() + backend.count("negative_hits" if envelope['value'] is None else "hits") + return envelope['value'] + except (ValueError, TypeError, UnicodeError): + backend.count("invalid_entries") + backend.call(client.delete, key) + if exact_id is not None and cfg.bloom: + bloom = f"{self.prefix}:{revision}:ids" + exists = backend.call(client.eval, BLOOM_CHECK, 2, bloom, bloom + ':ready', str(exact_id)) + if exists == 0: + backend.count("bloom_negatives") + return None + if backend.call(client.set, lock_key, owner, nx=True, px=cfg.lockMs): + break + backend.count("lock_waits") + if time.monotonic() >= end: + raise CacheBusy("Historical cache is rebuilding; retry later") + time.sleep(random.uniform(0.01, 0.025)) + try: + # Recheck after acquiring: a previous owner may have published between GET/SET. + cached = backend.call(client.get, key) + if cached is not None: + try: + envelope = json.loads(cached) + if not isinstance(envelope, dict) or set(envelope) != {'value'}: + raise ValueError() + backend.count("negative_hits" if envelope['value'] is None else "hits") + return envelope['value'] + except (ValueError, TypeError, UnicodeError): + backend.count("invalid_entries") + backend.call(client.delete, key) + self._rate('rebuild', cfg.rebuildsPerSecond) + result = self._compute(loader, revision) + payload = json.dumps({'value': result}, ensure_ascii=False, allow_nan=False).encode() + ttl = cfg.negativeTtlSeconds if exact_id is not None and result is None else cfg.ttlSeconds + ttl_ms = max(1, int(ttl * 1000 * random.uniform(0.8, 1.2))) + if len(payload) <= cfg.maxBytes: + try: + published = backend.call(client.eval, PUBLISH, 2, lock_key, key, owner, payload, ttl_ms) + except CacheUnavailable: + # A successful DB read is not repeated just because cache storage failed. + return result + if not published: + backend.count("lease_lost") + raise CacheBusy("Cache rebuild lease expired; retry later") + else: + backend.count("oversized") + backend.count("misses") + return result + finally: + try: + backend.call(client.eval, RELEASE, 1, lock_key, owner) + except CacheUnavailable: + pass + + def warm_bloom(self, identifiers): + """Build from authoritative IDs; publish only for an unchanged DB revision.""" + if not self.backend or not self.config.bloom: + return {"enabled": False} + backend, client = self.backend, self.backend.client + revision = self.db.revision(self.domain) + self._rate('rebuild', self.config.rebuildsPerSecond) + values = self._compute(identifiers, revision) + name = f"{self.prefix}:{revision}:ids" + temporary = name + ':' + secrets.token_hex(12) + try: + backend.call(client.eval, BLOOM_CREATE, 1, temporary, self.config.bloomErrorRate, + max(self.config.bloomCapacity, math.ceil(len(values) * 1.5))) + for start in range(0, len(values), 500): + backend.call(client.execute_command, 'BF.MADD', temporary, *values[start:start + 500]) + if self.db.revision(self.domain) != revision: + raise VersionChanged() + pipe = client.pipeline(transaction=True) + pipe.rename(temporary, name) + pipe.expire(name, 600) + pipe.set(name + ':ready', 'ready', ex=590) + backend.call(pipe.execute) + return {"enabled": True, "revision": revision, "ids": len(values), "ttl_seconds": 590} + finally: + try: + backend.call(client.delete, temporary) + except CacheUnavailable: + pass diff --git a/packages/dsh-triton-riscv/python/codex_agent/storage/database.py b/packages/dsh-triton-riscv/python/codex_agent/storage/database.py new file mode 100644 index 0000000..6a1adc0 --- /dev/null +++ b/packages/dsh-triton-riscv/python/codex_agent/storage/database.py @@ -0,0 +1,165 @@ +"""Bounded MySQL pools and workspace-scoped SQLAlchemy Core operations.""" +from contextlib import contextmanager +from contextvars import ContextVar +from functools import lru_cache +import hashlib +import os +from pathlib import Path + +from sqlalchemy import create_engine, select, update, delete, text +from sqlalchemy.dialects.mysql import insert +from sqlalchemy.engine import make_url +from sqlalchemy.exc import SQLAlchemyError + +from codex_agent.runtime_config import runtime_config +from .schema import counters, workspaces + + +class StorageError(RuntimeError): + pass + + +_read_connection = ContextVar("cache_read_connection", default=None) + + +@lru_cache(maxsize=8) +def _engine(raw, pool_size, overflow, timeout, pid): + try: + url = make_url(raw) + if url.drivername != "mysql+pymysql" or not url.database: + raise ValueError() + return create_engine(url, pool_size=pool_size, max_overflow=overflow, + pool_timeout=timeout, pool_pre_ping=True, pool_recycle=1800, + isolation_level="READ COMMITTED", hide_parameters=True, + connect_args={"charset": "utf8mb4", "connect_timeout": 5, + "read_timeout": 30, "write_timeout": 30}) + except (ValueError, SQLAlchemyError): + raise StorageError("Invalid MySQL URL; use mysql+pymysql and a database name") from None + + +def engine(): + cfg = runtime_config().storage + raw = os.environ.get(cfg.urlEnv, "") + if not raw: + raise StorageError(f"Set {cfg.urlEnv} to the MySQL connection URL, then run codex_agent.storage upgrade") + return _engine(raw, cfg.poolSize, cfg.maxOverflow, cfg.poolTimeout, os.getpid()) + + +class WorkspaceDatabase: + def __init__(self, root: Path): + self.root = Path(root).resolve() + self.workspace_id = hashlib.sha256(str(self.root).encode()).hexdigest() + self.engine = engine() + self.task_guard = None + self.cache_namespace = hashlib.sha256( + self.engine.url.render_as_string(hide_password=True).encode()).hexdigest()[:24] + + def predicate(self, table, *conditions): + return (table.c.workspace_id == self.workspace_id, *conditions) + + @contextmanager + def transaction(self): + borrowed = _read_connection.get() + if borrowed and borrowed[0] is self.engine: + try: + yield borrowed[1] + except SQLAlchemyError: + raise StorageError("MySQL cached read failed; check service and schema") from None + return + try: + with self.engine.begin() as connection: + connection.info["cache_changes"] = set() + try: + if self.task_guard is not None: + self.task_guard(connection) + yield connection + for workspace, domain in sorted(connection.info["cache_changes"]): + stmt = insert(counters).values(workspace_id=workspace, + name="cache-version:" + domain, value=1) + connection.execute(stmt.on_duplicate_key_update(value=counters.c.value + 1)) + finally: + connection.info.pop("cache_changes", None) + except SQLAlchemyError: + # Driver exceptions may include a username, host, SQL parameters or credentials. + raise StorageError("MySQL operation failed; check service, schema migration and constraints") from None + + def register(self): + with self.transaction() as conn: + stmt = insert(workspaces).values(workspace_id=self.workspace_id, root=str(self.root)) + conn.execute(stmt.on_duplicate_key_update(root=stmt.inserted.root)) + + def rows(self, table, *conditions, order=(), connection=None, limit=None, lock=False): + stmt = select(table).where(*self.predicate(table, *conditions)).order_by(*order) + if limit is not None: + stmt = stmt.limit(limit) + if lock: + stmt = stmt.with_for_update() + if connection is None: + with self.transaction() as conn: + return [dict(row) for row in conn.execute(stmt).mappings()] + return [dict(row) for row in connection.execute(stmt).mappings()] + + def insert(self, conn, table, values): + conn.execute(table.insert().values(**{**values, "workspace_id": self.workspace_id})) + self._changed(conn, table) + + def update(self, conn, table, values, *conditions): + result = conn.execute(update(table).where(*self.predicate(table, *conditions)).values(**values)) + if result.rowcount and set(values) != {"last_used_at"}: + self._changed(conn, table) + return result + + def delete(self, conn, table, *conditions): + result = conn.execute(delete(table).where(*self.predicate(table, *conditions))) + if result.rowcount: + self._changed(conn, table) + return result + + def _changed(self, conn, table): + domain = {"memories": "memory", "memory_chunks": "memory", + "references_catalog": "references"}.get(table.name) + if domain: + if "cache_changes" not in conn.info: + raise StorageError("Cached data writes require WorkspaceDatabase.transaction") + conn.info["cache_changes"].add((self.workspace_id, domain)) + + def revision(self, domain): + rows = self.rows(counters, counters.c.name == "cache-version:" + domain) + return rows[0]["value"] if rows else 0 + + @contextmanager + def cache_read_slot(self, limit): + """Server-wide, connection-owned slots survive Redis loss, not process loss.""" + from .cache import CacheBusy + acquired = None + token = None + try: + with self.engine.connect() as conn: + try: + for slot in range(limit): + name = f"triton-cache:{self.cache_namespace}:{slot}" + if conn.execute(text("SELECT GET_LOCK(:name, 0)"), {"name": name}).scalar() == 1: + acquired = name + break + if acquired is None: + raise CacheBusy("Historical retrieval is busy; retry later") + token = _read_connection.set((self.engine, conn)) + yield + finally: + if token is not None: + _read_connection.reset(token) + if acquired: + conn.execute(text("SELECT RELEASE_LOCK(:name)"), {"name": acquired}) + except SQLAlchemyError: + raise StorageError("MySQL cache read admission failed") from None + + def lock(self, conn, name): + stmt = insert(counters).values(workspace_id=self.workspace_id, name=name, value=0) + conn.execute(stmt.on_duplicate_key_update(value=counters.c.value)) + return conn.execute(select(counters.c.value).where( + *self.predicate(counters, counters.c.name == name)).with_for_update()).scalar_one() + + def allocate(self, conn, name): + value = self.lock(conn, name) + 1 + self.update(conn, counters, {"value": value}, counters.c.name == name) + return value diff --git a/packages/dsh-triton-riscv/python/codex_agent/storage/import_sqlite.py b/packages/dsh-triton-riscv/python/codex_agent/storage/import_sqlite.py new file mode 100644 index 0000000..6b10bf5 --- /dev/null +++ b/packages/dsh-triton-riscv/python/codex_agent/storage/import_sqlite.py @@ -0,0 +1,146 @@ +"""Import a stopped legacy SQLite workspace into a new MySQL workspace. + +Sources are opened read-only in a consistent SQLite read transaction. The MySQL +import commits once, refuses existing workspaces, and never rewrites evidence. +""" +from __future__ import annotations + +import argparse +from contextlib import ExitStack, contextmanager +import hashlib +import json +from pathlib import Path +import sqlite3 + +from codex_agent.storage.database import WorkspaceDatabase +from codex_agent.storage.schema import TABLES, workspaces, counters, meta +from codex_agent.memory import MemoryStore, SCHEMA_VERSION, memory_chunks + + +@contextmanager +def snapshot(path): + source = Path(path).resolve(strict=True) + connection = sqlite3.connect(source.as_uri() + "?mode=ro", uri=True) + connection.row_factory = sqlite3.Row + try: + connection.execute("PRAGMA query_only=ON") + connection.execute("BEGIN") + yield connection + finally: + connection.rollback() + connection.close() + + +def read_rows(connection, name): + if not connection.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (name,)).fetchone(): + return None + if name == "messages": + return [dict(row) for row in connection.execute('SELECT *, rowid AS ordinal FROM messages ORDER BY rowid')] + # Names come exclusively from the fixed schema, never a caller SQL fragment. + return [dict(row) for row in connection.execute('SELECT * FROM "' + name + '"')] + + +def import_sqlite(workspace: Path, *, platform=None, memory=None, references=None): + sources = {"platform": platform, "memory": memory, "references": references} + if not any(sources.values()): + raise ValueError("supply at least one SQLite source") + groups = {"platform": ("sessions", "messages", "runs", "events", "context_checkpoints"), + "memory": ("memories", "memory_chunks"), + "references": ("references_catalog",)} + data, provenance = {}, [] + old_version = None + with ExitStack() as stack: + for kind, source in sources.items(): + if source is None: + continue + source = Path(source).resolve(strict=True) + conn = stack.enter_context(snapshot(source)) + if kind == "memory": + versions = read_rows(conn, "metadata") or [] + old_version = next((int(r["value"]) for r in versions if r["key"] == "schema_version"), None) + if old_version is not None and old_version > SCHEMA_VERSION: + raise ValueError("source memory schema is newer than this importer") + for name in groups[kind]: + rows = read_rows(conn, name) + if rows is None and name not in {"context_checkpoints", "memory_chunks"}: + raise ValueError("missing required source table: " + name) + data[name] = rows or [] + provenance.append({"kind": kind, "path": str(source), + "main_file_sha256": hashlib.sha256(source.read_bytes()).hexdigest(), + "access": "read-only snapshot; committed WAL included"}) + if "memories" in data: + if old_version != SCHEMA_VERSION: + data["memory_chunks"] = [] + indexed = {row["memory_id"] for row in data["memory_chunks"]} + for row in data["memories"]: + if row["id"] in indexed: + continue + row.setdefault("embedding_json", None) + item = MemoryStore._row_dict(row) + for part in memory_chunks(item): + data["memory_chunks"].append(dict(memory_id=row["id"], kind=part.kind, + position=part.position, text=part.text, source_field=part.source_field)) + # Parent IDs/provenance stay unchanged. Old vectors describe a different layout. + for key in ("embedding_json", "embedding_provider", "embedding_model", "embedding_dim"): + row[key] = None + db = WorkspaceDatabase(workspace) + with db.transaction() as target: + if db.rows(workspaces, connection=target, lock=True): + raise ValueError("target workspace already exists; import into a new workspace, never overwrite") + db.insert(target, workspaces, {"root": str(db.root)}) + event_next = {} + for row in data.get("events", []): + event_next[row["run_id"]] = max(event_next.get(row["run_id"], 0), row["sequence"] + 1) + for row in data.get("runs", []): + row["next_sequence"] = event_next.get(row["id"], 0) + for name, rows in data.items(): + table = TABLES[name] + allowed = set(table.c.keys()) - {"workspace_id"} + for source_row in rows: + row = dict(source_row) + if set(row) - allowed: + raise ValueError("unrecognized source columns in " + name) + if name in {"memories", "memory_chunks"}: + for key in ("embedding_json", "embedding_provider", "embedding_model", "embedding_dim"): + row.setdefault(key, None) + if name == "memory_chunks": + row.setdefault("source_field", "") + if name == "sessions": + row.setdefault("pinned", 0) + db.insert(target, table, row) + actual = db.rows(table, connection=target) + if len(actual) != len(rows): + raise ValueError("import count mismatch in " + name) + # Verify every supplied field, not only counts or passed labels. + keys = [c.name for c in table.primary_key if c.name != "workspace_id"] + indexed = {tuple(r[k] for k in keys): r for r in actual} + for row in rows: + copy = indexed[tuple(row[k] for k in keys)] + if any(copy[k] != v for k, v in row.items()): + raise ValueError("import field mismatch in " + name) + for name, source, field in (("messages", "messages", "ordinal"), + ("events", "events", "id"), ("memories", "memories", "id")): + db.insert(target, counters, {"name": name, + "value": max((r[field] for r in data.get(source, [])), default=0)}) + if memory: + db.insert(target, meta, {"key": "schema_version", "value": str(SCHEMA_VERSION)}) + receipt = {"workspace": str(db.root), "workspace_id": db.workspace_id, + "sources": provenance, "counts": {k: len(v) for k, v in data.items()}, + "old_memory_version": old_version, "evidence_paths_rewritten": False, + "labels_reaudited": False} + db.insert(target, meta, {"key": "sqlite_import", "value": json.dumps(receipt, ensure_ascii=False)}) + return receipt + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--workspace", required=True, type=Path) + for key in ("platform", "memory", "references"): + parser.add_argument("--" + key, type=Path) + args = parser.parse_args() + print(json.dumps(import_sqlite(args.workspace, platform=args.platform, memory=args.memory, + references=args.references), ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/packages/dsh-triton-riscv/python/codex_agent/storage/migrations/env.py b/packages/dsh-triton-riscv/python/codex_agent/storage/migrations/env.py new file mode 100644 index 0000000..e14feef --- /dev/null +++ b/packages/dsh-triton-riscv/python/codex_agent/storage/migrations/env.py @@ -0,0 +1,7 @@ +from alembic import context +from codex_agent.storage.schema import metadata + +connection = context.config.attributes["connection"] +context.configure(connection=connection, target_metadata=metadata) +with context.begin_transaction(): + context.run_migrations() diff --git a/packages/dsh-triton-riscv/python/codex_agent/storage/migrations/versions/0001_mysql.py b/packages/dsh-triton-riscv/python/codex_agent/storage/migrations/versions/0001_mysql.py new file mode 100644 index 0000000..d58922e --- /dev/null +++ b/packages/dsh-triton-riscv/python/codex_agent/storage/migrations/versions/0001_mysql.py @@ -0,0 +1,162 @@ +"""Initial workspace-isolated MySQL schema, frozen at revision 0001.""" +from alembic import op +from sqlalchemy import inspect + +revision = "0001_mysql" +down_revision = None + +TABLE_DDL = ( + """CREATE TABLE IF NOT EXISTS counters ( + workspace_id VARCHAR(64) NOT NULL, + name VARCHAR(100) NOT NULL, + value BIGINT NOT NULL, + PRIMARY KEY (workspace_id, name) +)ENGINE=InnoDB CHARSET=utf8mb4 COLLATE utf8mb4_bin""", + """CREATE TABLE IF NOT EXISTS memories ( + workspace_id VARCHAR(64) NOT NULL, + id BIGINT NOT NULL, + fingerprint VARCHAR(64) NOT NULL, + memory_type VARCHAR(100) NOT NULL, + operator VARCHAR(255) NOT NULL, + semantics LONGTEXT NOT NULL, + pytorch_reference LONGTEXT NOT NULL, + tl_ops_json LONGTEXT NOT NULL, + failure_stage VARCHAR(100), + error_signature LONGTEXT, + environment_json LONGTEXT NOT NULL, + summary LONGTEXT NOT NULL, + searchable_text LONGTEXT NOT NULL, + outcome VARCHAR(64) NOT NULL, + evidence_json LONGTEXT NOT NULL, + confidence_grade VARCHAR(8) NOT NULL, + confidence FLOAT(53) NOT NULL, + source_run LONGTEXT NOT NULL, + test_sha256 VARCHAR(64), + active INTEGER NOT NULL DEFAULT '1', + archive_reason LONGTEXT, + created_at VARCHAR(40) NOT NULL, + updated_at VARCHAR(40) NOT NULL, + last_used_at VARCHAR(40), + useful_count BIGINT NOT NULL DEFAULT '0', + embedding_json LONGTEXT, + embedding_provider VARCHAR(255), + embedding_model VARCHAR(255), + embedding_dim INTEGER, + PRIMARY KEY (workspace_id, id), + UNIQUE (workspace_id, fingerprint) +)ENGINE=InnoDB CHARSET=utf8mb4 COLLATE utf8mb4_bin""", + """CREATE TABLE IF NOT EXISTS metadata ( + workspace_id VARCHAR(64) NOT NULL, + `key` VARCHAR(100) NOT NULL, + value LONGTEXT NOT NULL, + PRIMARY KEY (workspace_id, `key`) +)ENGINE=InnoDB CHARSET=utf8mb4 COLLATE utf8mb4_bin""", + """CREATE TABLE IF NOT EXISTS references_catalog ( + workspace_id VARCHAR(64) NOT NULL, + id VARCHAR(64) NOT NULL, + admitted INTEGER NOT NULL, + payload LONGTEXT NOT NULL, + PRIMARY KEY (workspace_id, id) +)ENGINE=InnoDB CHARSET=utf8mb4 COLLATE utf8mb4_bin""", + """CREATE TABLE IF NOT EXISTS sessions ( + workspace_id VARCHAR(64) NOT NULL, + id VARCHAR(64) NOT NULL, + title LONGTEXT NOT NULL, + status VARCHAR(64) NOT NULL, + created_at VARCHAR(40) NOT NULL, + updated_at VARCHAR(40) NOT NULL, + pinned INTEGER NOT NULL DEFAULT '0', + PRIMARY KEY (workspace_id, id) +)ENGINE=InnoDB CHARSET=utf8mb4 COLLATE utf8mb4_bin""", + """CREATE TABLE IF NOT EXISTS workspaces ( + workspace_id VARCHAR(64) NOT NULL, + root LONGTEXT NOT NULL, + PRIMARY KEY (workspace_id) +)ENGINE=InnoDB CHARSET=utf8mb4 COLLATE utf8mb4_bin""", + """CREATE TABLE IF NOT EXISTS context_checkpoints ( + workspace_id VARCHAR(64) NOT NULL, + session_id VARCHAR(64) NOT NULL, + summary LONGTEXT NOT NULL, + through_message_id VARCHAR(64), + source_ids_json LONGTEXT NOT NULL, + metrics_json LONGTEXT NOT NULL, + updated_at VARCHAR(40) NOT NULL, + PRIMARY KEY (workspace_id, session_id), + FOREIGN KEY(workspace_id, session_id) REFERENCES sessions (workspace_id, id) ON DELETE CASCADE +)ENGINE=InnoDB CHARSET=utf8mb4 COLLATE utf8mb4_bin""", + """CREATE TABLE IF NOT EXISTS memory_chunks ( + workspace_id VARCHAR(64) NOT NULL, + memory_id BIGINT NOT NULL, + kind VARCHAR(64) NOT NULL, + position INTEGER NOT NULL, + text LONGTEXT NOT NULL, + source_field LONGTEXT NOT NULL, + embedding_json LONGTEXT, + embedding_provider VARCHAR(255), + embedding_model VARCHAR(255), + embedding_dim INTEGER, + PRIMARY KEY (workspace_id, memory_id, kind, position), + FOREIGN KEY(workspace_id, memory_id) REFERENCES memories (workspace_id, id) ON DELETE CASCADE +)ENGINE=InnoDB CHARSET=utf8mb4 COLLATE utf8mb4_bin""", + """CREATE TABLE IF NOT EXISTS messages ( + workspace_id VARCHAR(64) NOT NULL, + id VARCHAR(64) NOT NULL, + session_id VARCHAR(64) NOT NULL, + ordinal BIGINT NOT NULL, + `role` VARCHAR(64) NOT NULL, + content LONGTEXT NOT NULL, + metadata_json LONGTEXT NOT NULL, + created_at VARCHAR(40) NOT NULL, + PRIMARY KEY (workspace_id, id), + FOREIGN KEY(workspace_id, session_id) REFERENCES sessions (workspace_id, id) ON DELETE CASCADE +)ENGINE=InnoDB CHARSET=utf8mb4 COLLATE utf8mb4_bin""", + """CREATE TABLE IF NOT EXISTS runs ( + workspace_id VARCHAR(64) NOT NULL, + id VARCHAR(64) NOT NULL, + session_id VARCHAR(64) NOT NULL, + intent LONGTEXT NOT NULL, + operator VARCHAR(255), + status VARCHAR(64) NOT NULL, + phase VARCHAR(64) NOT NULL, + request_json LONGTEXT NOT NULL, + result_json LONGTEXT NOT NULL, + created_at VARCHAR(40) NOT NULL, + updated_at VARCHAR(40) NOT NULL, + next_sequence BIGINT NOT NULL DEFAULT '0', + PRIMARY KEY (workspace_id, id), + FOREIGN KEY(workspace_id, session_id) REFERENCES sessions (workspace_id, id) ON DELETE CASCADE +)ENGINE=InnoDB CHARSET=utf8mb4 COLLATE utf8mb4_bin""", + """CREATE TABLE IF NOT EXISTS events ( + workspace_id VARCHAR(64) NOT NULL, + id BIGINT NOT NULL, + run_id VARCHAR(64) NOT NULL, + sequence BIGINT NOT NULL, + event_type VARCHAR(100) NOT NULL, + payload_json LONGTEXT NOT NULL, + created_at VARCHAR(40) NOT NULL, + PRIMARY KEY (workspace_id, id), + UNIQUE (workspace_id, run_id, sequence), + FOREIGN KEY(workspace_id, run_id) REFERENCES runs (workspace_id, id) ON DELETE CASCADE +)ENGINE=InnoDB CHARSET=utf8mb4 COLLATE utf8mb4_bin""", +) +INDEX_DDL = ( + ("memories", "idx_memories_active", "CREATE INDEX idx_memories_active ON memories (workspace_id, active)"), + ("memories", "idx_memories_operator", "CREATE INDEX idx_memories_operator ON memories (workspace_id, operator)"), + ("memories", "idx_memories_stage", "CREATE INDEX idx_memories_stage ON memories (workspace_id, failure_stage)"), + ("messages", "idx_messages_session", "CREATE INDEX idx_messages_session ON messages (workspace_id, session_id, ordinal)"), + ("runs", "idx_runs_session", "CREATE INDEX idx_runs_session ON runs (workspace_id, session_id, created_at)"), +) + + +def upgrade(): + # MySQL DDL commits independently; retry completes a partially applied revision. + for statement in TABLE_DDL: + op.execute(statement) + inspector = inspect(op.get_bind()) + for table, name, statement in INDEX_DDL: + if name not in {item["name"] for item in inspector.get_indexes(table)}: + op.execute(statement) + + +def downgrade(): + raise RuntimeError("Destructive downgrade disabled; restore a verified backup instead") diff --git a/packages/dsh-triton-riscv/python/codex_agent/storage/migrations/versions/0002_tasks.py b/packages/dsh-triton-riscv/python/codex_agent/storage/migrations/versions/0002_tasks.py new file mode 100644 index 0000000..908ffd7 --- /dev/null +++ b/packages/dsh-triton-riscv/python/codex_agent/storage/migrations/versions/0002_tasks.py @@ -0,0 +1,38 @@ +"""Transactional jobs and delivery ledger, frozen at revision 0002.""" +from alembic import op +from sqlalchemy import inspect + +revision = "0002_tasks" +down_revision = "0001_mysql" +TABLE_DDL = [ + "\nCREATE TABLE IF NOT EXISTS jobs (\n\tworkspace_id VARCHAR(64) NOT NULL, \n\tid VARCHAR(64) NOT NULL, \n\tkind VARCHAR(24) NOT NULL, \n\tstatus VARCHAR(32) NOT NULL, \n\tidempotency_key VARCHAR(64) NOT NULL, \n\tfingerprint VARCHAR(64) NOT NULL, \n\tpayload_json LONGTEXT NOT NULL, \n\tresult_json LONGTEXT NOT NULL, \n\trun_id VARCHAR(64), \n\texecution_host VARCHAR(255) NOT NULL, \n\tattempt INTEGER NOT NULL, \n\tgeneration INTEGER NOT NULL, \n\towner VARCHAR(64), \n\tlease_until BIGINT NOT NULL, \n\teffects_started INTEGER NOT NULL, \n\tcancel_requested INTEGER NOT NULL, \n\tcreated_ms BIGINT NOT NULL, \n\tupdated_ms BIGINT NOT NULL, \n\tPRIMARY KEY (workspace_id, id), \n\tUNIQUE (workspace_id, idempotency_key)\n)ENGINE=InnoDB CHARSET=utf8mb4 COLLATE utf8mb4_bin\n\n", + "\nCREATE TABLE IF NOT EXISTS job_attempts (\n\tworkspace_id VARCHAR(64) NOT NULL, \n\tjob_id VARCHAR(64) NOT NULL, \n\tnumber INTEGER NOT NULL, \n\towner VARCHAR(64) NOT NULL, \n\tstatus VARCHAR(32) NOT NULL, \n\tstarted_ms BIGINT NOT NULL, \n\tfinished_ms BIGINT, \n\tresult_json LONGTEXT NOT NULL, \n\tPRIMARY KEY (workspace_id, job_id, number)\n)ENGINE=InnoDB CHARSET=utf8mb4 COLLATE utf8mb4_bin\n\n", + "\nCREATE TABLE IF NOT EXISTS outbox_events (\n\tworkspace_id VARCHAR(64) NOT NULL, \n\tid VARCHAR(64) NOT NULL, \n\tjob_id VARCHAR(64) NOT NULL, \n\tgeneration INTEGER NOT NULL, \n\tkind VARCHAR(24) NOT NULL, \n\tstatus VARCHAR(24) NOT NULL, \n\tattempts INTEGER NOT NULL, \n\towner VARCHAR(64), \n\tlease_until BIGINT NOT NULL, \n\tavailable_ms BIGINT NOT NULL, \n\tcreated_ms BIGINT NOT NULL, \n\tpublished_ms BIGINT, \n\tlast_error VARCHAR(255), \n\tPRIMARY KEY (workspace_id, id), \n\tUNIQUE (workspace_id, job_id, generation)\n)ENGINE=InnoDB CHARSET=utf8mb4 COLLATE utf8mb4_bin\n\n", + "\nCREATE TABLE IF NOT EXISTS inbox_events (\n\tworkspace_id VARCHAR(64) NOT NULL, \n\tevent_id VARCHAR(64) NOT NULL, \n\tjob_id VARCHAR(64) NOT NULL, \n\tstatus VARCHAR(24) NOT NULL, \n\treceived_ms BIGINT NOT NULL, \n\tfinished_ms BIGINT, \n\tPRIMARY KEY (workspace_id, event_id)\n)ENGINE=InnoDB CHARSET=utf8mb4 COLLATE utf8mb4_bin\n\n", + "\nCREATE TABLE IF NOT EXISTS task_artifacts (\n\tworkspace_id VARCHAR(64) NOT NULL, \n\tid VARCHAR(64) NOT NULL, \n\tjob_id VARCHAR(64) NOT NULL, \n\tattempt INTEGER NOT NULL, \n\tsha256 VARCHAR(64) NOT NULL, \n\tsize BIGINT NOT NULL, \n\tname VARCHAR(120) NOT NULL, \n\tstorage_key LONGTEXT NOT NULL, \n\tPRIMARY KEY (workspace_id, id)\n)ENGINE=InnoDB CHARSET=utf8mb4 COLLATE utf8mb4_bin\n\n" +] +INDEX_DDL = [ + [ + "jobs", + "idx_job_recovery", + "CREATE INDEX idx_job_recovery ON jobs (workspace_id, status, lease_until)" + ], + [ + "outbox_events", + "idx_outbox_ready", + "CREATE INDEX idx_outbox_ready ON outbox_events (workspace_id, status, available_ms)" + ] +] + + +def upgrade(): + for statement in TABLE_DDL: + op.execute(statement) + inspector = inspect(op.get_bind()) + for table, name, statement in INDEX_DDL: + if name not in {item["name"] for item in inspector.get_indexes(table)}: + op.execute(statement) + + +def downgrade(): + raise RuntimeError("Destructive downgrade disabled; restore a verified backup instead") diff --git a/packages/dsh-triton-riscv/python/codex_agent/storage/schema.py b/packages/dsh-triton-riscv/python/codex_agent/storage/schema.py new file mode 100644 index 0000000..be05166 --- /dev/null +++ b/packages/dsh-triton-riscv/python/codex_agent/storage/schema.py @@ -0,0 +1,94 @@ +"""Version-one schema. Every domain key is scoped to a canonical workspace.""" +from sqlalchemy import (BigInteger, Column, Float, ForeignKeyConstraint, Index, + Integer, MetaData, String, Table, UniqueConstraint) +from sqlalchemy.dialects.mysql import LONGTEXT + +metadata = MetaData() + + +def table(name, *columns): + return Table(name, metadata, Column("workspace_id", String(64), primary_key=True), + *columns, mysql_engine="InnoDB", mysql_charset="utf8mb4", + mysql_collate="utf8mb4_bin") + + +def text(name, nullable=False, default=None): + return Column(name, LONGTEXT, nullable=nullable, server_default=default) + + +def timestamp(name): + # Preserve historical timestamp text verbatim, including its timezone. + return Column(name, String(40), nullable=False) + + +workspaces = table("workspaces", text("root")) +counters = table("counters", Column("name", String(100), primary_key=True), + Column("value", BigInteger, nullable=False)) +meta = table("metadata", Column("key", String(100), primary_key=True), text("value")) +sessions = table("sessions", Column("id", String(64), primary_key=True), + text("title"), Column("status", String(64), nullable=False), + timestamp("created_at"), timestamp("updated_at"), + Column("pinned", Integer, nullable=False, server_default="0")) +messages = table("messages", Column("id", String(64), primary_key=True), + Column("session_id", String(64), nullable=False), + Column("ordinal", BigInteger, nullable=False), + Column("role", String(64), nullable=False), text("content"), + text("metadata_json"), timestamp("created_at"), + ForeignKeyConstraint(["workspace_id", "session_id"], + ["sessions.workspace_id", "sessions.id"], ondelete="CASCADE")) +runs = table("runs", Column("id", String(64), primary_key=True), + Column("session_id", String(64), nullable=False), text("intent"), + Column("operator", String(255)), Column("status", String(64), nullable=False), + Column("phase", String(64), nullable=False), text("request_json"), text("result_json"), + timestamp("created_at"), timestamp("updated_at"), + Column("next_sequence", BigInteger, nullable=False, server_default="0"), + ForeignKeyConstraint(["workspace_id", "session_id"], + ["sessions.workspace_id", "sessions.id"], ondelete="CASCADE")) +events = table("events", Column("id", BigInteger, primary_key=True, autoincrement=False), + Column("run_id", String(64), nullable=False), + Column("sequence", BigInteger, nullable=False), + Column("event_type", String(100), nullable=False), text("payload_json"), + timestamp("created_at"), + UniqueConstraint("workspace_id", "run_id", "sequence"), + ForeignKeyConstraint(["workspace_id", "run_id"], + ["runs.workspace_id", "runs.id"], ondelete="CASCADE")) +checkpoints = table("context_checkpoints", Column("session_id", String(64), primary_key=True), + text("summary"), Column("through_message_id", String(64)), + text("source_ids_json"), text("metrics_json"), timestamp("updated_at"), + ForeignKeyConstraint(["workspace_id", "session_id"], + ["sessions.workspace_id", "sessions.id"], ondelete="CASCADE")) +memories = table("memories", Column("id", BigInteger, primary_key=True, autoincrement=False), + Column("fingerprint", String(64), nullable=False), + Column("memory_type", String(100), nullable=False), + Column("operator", String(255), nullable=False), text("semantics"), + text("pytorch_reference"), text("tl_ops_json"), Column("failure_stage", String(100)), + text("error_signature", nullable=True), text("environment_json"), text("summary"), + text("searchable_text"), Column("outcome", String(64), nullable=False), + text("evidence_json"), Column("confidence_grade", String(8), nullable=False), + Column("confidence", Float(53), nullable=False), text("source_run"), + Column("test_sha256", String(64)), + Column("active", Integer, nullable=False, server_default="1"), + text("archive_reason", nullable=True), timestamp("created_at"), timestamp("updated_at"), + Column("last_used_at", String(40)), + Column("useful_count", BigInteger, nullable=False, server_default="0"), + text("embedding_json", nullable=True), Column("embedding_provider", String(255)), + Column("embedding_model", String(255)), Column("embedding_dim", Integer), + UniqueConstraint("workspace_id", "fingerprint")) +chunks = table("memory_chunks", Column("memory_id", BigInteger, primary_key=True, autoincrement=False), + Column("kind", String(64), primary_key=True), + Column("position", Integer, primary_key=True, autoincrement=False), text("text"), + text("source_field"), text("embedding_json", nullable=True), + Column("embedding_provider", String(255)), Column("embedding_model", String(255)), + Column("embedding_dim", Integer), + ForeignKeyConstraint(["workspace_id", "memory_id"], + ["memories.workspace_id", "memories.id"], ondelete="CASCADE")) +references = table("references_catalog", Column("id", String(64), primary_key=True), + Column("admitted", Integer, nullable=False), text("payload")) + +Index("idx_messages_session", messages.c.workspace_id, messages.c.session_id, messages.c.ordinal) +Index("idx_runs_session", runs.c.workspace_id, runs.c.session_id, runs.c.created_at) +Index("idx_memories_active", memories.c.workspace_id, memories.c.active) +Index("idx_memories_operator", memories.c.workspace_id, memories.c.operator) +Index("idx_memories_stage", memories.c.workspace_id, memories.c.failure_stage) + +TABLES = {t.name: t for t in metadata.tables.values()} diff --git a/packages/dsh-triton-riscv/python/codex_agent/tasks/__init__.py b/packages/dsh-triton-riscv/python/codex_agent/tasks/__init__.py new file mode 100644 index 0000000..04a052d --- /dev/null +++ b/packages/dsh-triton-riscv/python/codex_agent/tasks/__init__.py @@ -0,0 +1 @@ +"""Durable task application layer shared by HTTP, CLI and plugin adapters.""" diff --git a/packages/dsh-triton-riscv/python/codex_agent/tasks/context.py b/packages/dsh-triton-riscv/python/codex_agent/tasks/context.py new file mode 100644 index 0000000..555822d --- /dev/null +++ b/packages/dsh-triton-riscv/python/codex_agent/tasks/context.py @@ -0,0 +1,4 @@ +"""Internal execution context, not a model-controlled or ambient environment flag.""" +from contextvars import ContextVar + +defer_memory = ContextVar("worker_defers_memory", default=False) diff --git a/packages/dsh-triton-riscv/python/codex_agent/tasks/schema.py b/packages/dsh-triton-riscv/python/codex_agent/tasks/schema.py new file mode 100644 index 0000000..fdd9a95 --- /dev/null +++ b/packages/dsh-triton-riscv/python/codex_agent/tasks/schema.py @@ -0,0 +1,36 @@ +"""Queue schema, kept independent of conversation and memory models.""" +from sqlalchemy import BigInteger, Column, Index, Integer, String, UniqueConstraint +from codex_agent.storage.schema import table, text + +jobs = table("jobs", Column("id", String(64), primary_key=True), + Column("kind", String(24), nullable=False), Column("status", String(32), nullable=False), + Column("idempotency_key", String(64), nullable=False), Column("fingerprint", String(64), nullable=False), + text("payload_json"), text("result_json"), Column("run_id", String(64)), + Column("execution_host", String(255), nullable=False), + Column("attempt", Integer, nullable=False), Column("generation", Integer, nullable=False), + Column("owner", String(64)), Column("lease_until", BigInteger, nullable=False), + Column("effects_started", Integer, nullable=False), Column("cancel_requested", Integer, nullable=False), + Column("created_ms", BigInteger, nullable=False), Column("updated_ms", BigInteger, nullable=False), + UniqueConstraint("workspace_id", "idempotency_key")) +attempts = table("job_attempts", Column("job_id", String(64), primary_key=True), + Column("number", Integer, primary_key=True, autoincrement=False), + Column("owner", String(64), nullable=False), Column("status", String(32), nullable=False), + Column("started_ms", BigInteger, nullable=False), Column("finished_ms", BigInteger), text("result_json")) +outbox = table("outbox_events", Column("id", String(64), primary_key=True), + Column("job_id", String(64), nullable=False), Column("generation", Integer, nullable=False), + Column("kind", String(24), nullable=False), Column("status", String(24), nullable=False), + Column("attempts", Integer, nullable=False), Column("owner", String(64)), + Column("lease_until", BigInteger, nullable=False), Column("available_ms", BigInteger, nullable=False), + Column("created_ms", BigInteger, nullable=False), Column("published_ms", BigInteger), + Column("last_error", String(255)), UniqueConstraint("workspace_id", "job_id", "generation")) +inbox = table("inbox_events", Column("event_id", String(64), primary_key=True), + Column("job_id", String(64), nullable=False), Column("status", String(24), nullable=False), + Column("received_ms", BigInteger, nullable=False), Column("finished_ms", BigInteger)) +artifacts = table("task_artifacts", Column("id", String(64), primary_key=True), + Column("job_id", String(64), nullable=False), Column("attempt", Integer, nullable=False), + Column("sha256", String(64), nullable=False), Column("size", BigInteger, nullable=False), + Column("name", String(120), nullable=False), text("storage_key")) +Index("idx_job_recovery", jobs.c.workspace_id, jobs.c.status, jobs.c.lease_until) +Index("idx_outbox_ready", outbox.c.workspace_id, outbox.c.status, outbox.c.available_ms) + +QUEUE_TABLES = (jobs, attempts, outbox, inbox, artifacts) diff --git a/packages/dsh-triton-riscv/python/codex_agent/tasks/service.py b/packages/dsh-triton-riscv/python/codex_agent/tasks/service.py new file mode 100644 index 0000000..72d71e9 --- /dev/null +++ b/packages/dsh-triton-riscv/python/codex_agent/tasks/service.py @@ -0,0 +1,78 @@ +"""Submission validates capabilities and binds inputs before creating durable work.""" +import json +import uuid + +from codex_agent.execution_guard import approval_digest, artifact +from codex_agent.runtime_config import permission_enabled, runtime_config, remote_execution_identity +from codex_agent.platform.store import timestamp +from codex_agent.storage.schema import messages, runs, sessions +from .store import JobStore, Conflict, encode, digest + + +def request_key(kind, key): + if not isinstance(key, str) or not 1 <= len(key) <= 160: + raise ValueError("request_id must contain 1..160 characters") + # User keys cannot occupy internal follow-up identities. + return "user:" + kind + ":" + digest(key) + + +class TaskService: + def __init__(self, root): + self.root = root.resolve() + self.jobs = JobStore(self.root) + + def submit_validation(self, operator, approved_run_id, key): + key = request_key("validation", key) + if not runtime_config().queue.enabled: + raise ValueError("queue execution is disabled") + if not permission_enabled("validation"): + raise PermissionError("validation capability is disabled") + plan = artifact(self.root, "validation", approved_run_id) + if plan.get("operator") != operator or plan.get("approval", {}).get("status") != "approved": + raise PermissionError("a matching host-approved validation plan is required") + seal = approval_digest(self.root, "validation", plan) + if plan.get("approval_digest") != seal: + raise PermissionError("approved plan changed; request approval again") + return self.jobs.submit("validation", {"operator": operator, "approved_run_id": approved_run_id, + "approval_digest": seal, "source_env": plan.get("source_env", True), + "timeout_seconds": plan.get("timeout_seconds", 900), "remote_identity": remote_execution_identity()}, key) + + def submit_turn(self, platform, session_id, text, key): + key = request_key("agent", key) + if not runtime_config().queue.enabled: + raise ValueError("queue execution is disabled") + if not text.strip() or "\x00" in text: + raise ValueError("invalid task text") + db, store = self.jobs.db, platform.store + with db.transaction() as conn: + db.lock(conn, "task-submit") + old = self.jobs.find_key(conn, key) + if old: + payload = json.loads(old["payload_json"]) + if old["kind"] != "agent" or payload.get("session_id") != session_id or payload.get("text") != text: + raise Conflict("idempotency key reused for a different message") + run = store._get(runs, old["run_id"], connection=conn) + message_id = json.loads(run["request_json"])["user_message_id"] + message = store._get(messages, message_id, connection=conn) + else: + session = store._get(sessions, session_id, connection=conn, lock=True) + now, run_id, message_id = timestamp(), uuid.uuid4().hex, uuid.uuid4().hex + message = dict(id=message_id, session_id=session_id, ordinal=db.allocate(conn, "messages"), + role="user", content=text, metadata_json="{}", created_at=now) + db.insert(conn, messages, message) + db.update(conn, sessions, {"updated_at": now, "title": text[:80] if session["title"] == "新对话" else session["title"]}, sessions.c.id == session_id) + request = {"action": "deepseek-harness", "task": text, + "harness_session_id": "triton-job-" + run_id, "user_message_id": message_id, + "requires_confirmation": False} + run = dict(id=run_id, session_id=session_id, intent="harness-agent", operator=None, + status="queued", phase="outbox-pending", request_json=encode(request), result_json="{}", + created_at=now, updated_at=now, next_sequence=0) + db.insert(conn, runs, run) + submitted = self.jobs.submit("agent", {"run_id": run_id, "session_id": session_id, "text": text}, + key, run_id=run_id, connection=conn) + old = {"id": submitted["id"]} + store._add_event(conn, run_id, "queued", {"message": "Task persisted for RabbitMQ delivery"}) + return {"user_message": store._row(message), "assistant_message": None, + "task": {"intent": "harness-agent", "operator": None, "confidence": 1.0, + "runtime": "deepseek-harness"}, "run": store.get_run(run["id"]), + "job": self.jobs.get(old["id"])} diff --git a/packages/dsh-triton-riscv/python/codex_agent/tasks/store.py b/packages/dsh-triton-riscv/python/codex_agent/tasks/store.py new file mode 100644 index 0000000..146e66d --- /dev/null +++ b/packages/dsh-triton-riscv/python/codex_agent/tasks/store.py @@ -0,0 +1,297 @@ +"""Transactional task/outbox records. AMQP contains references, never authority.""" +from contextlib import nullcontext +import hashlib +import json +import secrets +import socket + +from sqlalchemy import and_, or_, select, func + +from codex_agent.runtime_config import runtime_config +from codex_agent.storage.database import WorkspaceDatabase +from codex_agent.platform.store import PlatformStore, timestamp +from codex_agent.storage.schema import runs +from .schema import jobs, attempts, outbox, inbox, artifacts + + +TERMINAL = {"succeeded", "failed", "cancelled", "dead"} + + +class Conflict(ValueError): + pass + + +class LeaseLost(RuntimeError): + pass + + +def encode(value): + return json.dumps(value, sort_keys=True, ensure_ascii=False, separators=(",", ":"), allow_nan=False) + + +def digest(value): + return hashlib.sha256(encode(value).encode()).hexdigest() + + +def now_ms(conn): + # One authority for leases even when clients have different wall clocks. + return int(conn.execute(select(func.unix_timestamp(func.current_timestamp(3)) * 1000)).scalar_one()) + + +class JobStore: + def __init__(self, root): + self.db = WorkspaceDatabase(root) + self.db.register() + self.platform = PlatformStore(root) + self.config = runtime_config().queue + + def _row(self, conn, table, identifier, *, lock=False): + rows = self.db.rows(table, table.c.id == identifier, connection=conn, lock=lock) + if not rows: + raise KeyError("unknown task record") + return rows[0] + + def get(self, identifier): + with self.db.transaction() as conn: + row = self._row(conn, jobs, identifier) + return self.public(row) + + @staticmethod + def public(row): + result = {k: v for k, v in row.items() if k not in {"owner", "workspace_id", "idempotency_key"}} + for field in ("payload", "result"): + result[field] = json.loads(result.pop(field + "_json")) + return result + + def find_key(self, conn, key): + return next(iter(self.db.rows(jobs, jobs.c.idempotency_key == digest(key), connection=conn)), None) + + def submit(self, kind, payload, key, *, run_id=None, connection=None): + if kind not in {"agent", "validation", "memory"}: + raise ValueError("unsupported job kind") + if not isinstance(key, str) or not 1 <= len(key) <= 160: + raise ValueError("idempotency key must contain 1..160 characters") + serialized = encode(payload) + if len(serialized.encode()) > 32768: + raise ValueError("task payload exceeds 32 KiB") + fingerprint = digest([kind, payload]) + with nullcontext(connection) if connection is not None else self.db.transaction() as conn: + self.db.lock(conn, "task-submit") + previous = self.find_key(conn, key) + if previous: + if previous["fingerprint"] != fingerprint: + raise Conflict("idempotency key reused with different task input") + return self.public(previous) + count = conn.execute(select(func.count()).select_from(jobs).where( + *self.db.predicate(jobs, jobs.c.status.not_in(TERMINAL)))).scalar_one() + if kind != "memory" and count >= self.config.maxPending: + raise Conflict("workspace task capacity reached") + latest = conn.execute(select(func.max(jobs.c.created_ms)).where(*self.db.predicate(jobs))).scalar_one() + now, identifier = max(now_ms(conn), (latest or 0) + 1), secrets.token_hex(16) + value = dict(id=identifier, kind=kind, status="queued", idempotency_key=digest(key), + fingerprint=fingerprint, payload_json=serialized, result_json="{}", run_id=run_id, + execution_host=socket.gethostname(), attempt=0, generation=1, owner=None, + lease_until=0, effects_started=0, cancel_requested=0, created_ms=now, updated_ms=now) + self.db.insert(conn, jobs, value) + self._enqueue(conn, value, now) + return self.public(value) + + def _enqueue(self, conn, job, at): + self.db.insert(conn, outbox, dict(id=secrets.token_hex(16), job_id=job["id"], + generation=job["generation"], kind=job["kind"], status="pending", attempts=0, + owner=None, lease_until=0, available_ms=at, created_ms=now_ms(conn), + published_ms=None, last_error=None)) + + def claim_publication(self): + with self.db.transaction() as conn: + now = now_ms(conn) + stmt = select(outbox).where(*self.db.predicate(outbox, + outbox.c.available_ms <= now, or_(outbox.c.status == "pending", + and_(outbox.c.status == "publishing", outbox.c.lease_until < now)))) + row = conn.execute(stmt.order_by(outbox.c.created_ms, outbox.c.id) + .limit(1).with_for_update(skip_locked=True)).mappings().first() + if row is None: + return None + owner = secrets.token_hex(16) + self.db.update(conn, outbox, dict(status="publishing", owner=owner, + lease_until=now + 30000, attempts=row["attempts"] + 1), outbox.c.id == row["id"]) + return {**dict(row), "owner": owner} + + def publication_result(self, event, error=None): + with self.db.transaction() as conn: + row = self._row(conn, outbox, event["id"], lock=True) + now = now_ms(conn) + if row["owner"] != event["owner"] or row["status"] != "publishing" or row["lease_until"] < now: + raise LeaseLost("outbox lease expired") + # Keep failed deliveries visible and retryable; never discard on broker downtime. + values = dict(owner=None, lease_until=0, status="pending" if error else "published", + last_error=error, available_ms=now + min(60000, 1000 * 2 ** min(row["attempts"], 6)), + published_ms=None if error else now) + self.db.update(conn, outbox, values, outbox.c.id == row["id"]) + + def envelope(self, event): + return {"version": 1, "workspace_id": self.db.workspace_id, "event_id": event["id"], + "job_id": event["job_id"], "kind": event["kind"], "generation": event["generation"]} + + def claim(self, envelope, kind): + if (not isinstance(envelope, dict) or set(envelope) != + {"version", "workspace_id", "event_id", "job_id", "kind", "generation"} + or type(envelope["version"]) is not int or envelope["version"] != 1 + or envelope["workspace_id"] != self.db.workspace_id or envelope["kind"] != kind): + raise ValueError("invalid task envelope") + with self.db.transaction() as conn: + self.db.lock(conn, "task-claim") + event = self._row(conn, outbox, envelope["event_id"]) + if self.envelope(event) != envelope: + raise ValueError("message does not match the durable outbox") + job = self._row(conn, jobs, envelope["job_id"], lock=True) + if job["execution_host"] != socket.gethostname(): + self.db.update(conn, jobs, {"result_json": encode({"reason": "worker host mismatch", "expected_host": job["execution_host"]})}, jobs.c.id == job["id"]) + if job["status"] == "queued" and job["generation"] == envelope["generation"]: + self._retry_queued(conn, job, 30000) + return None + if job["status"] in TERMINAL or job["generation"] != envelope["generation"]: + return None + if job["status"] != "queued": + return None # Recovery, not redelivery, owns an expired/uncertain attempt. + blockers = self.db.rows(jobs, jobs.c.id != job["id"], + jobs.c.kind.in_(["agent", "validation"]), + or_(jobs.c.status.in_(["running", "needs-reconciliation"]), + and_(jobs.c.status == "queued", or_(jobs.c.created_ms < job["created_ms"], + and_(jobs.c.created_ms == job["created_ms"], jobs.c.id < job["id"])))), connection=conn) + if blockers and kind != "memory": + self._retry_queued(conn, job, 1000) + return None + now, owner = now_ms(conn), secrets.token_hex(16) + values = dict(status="running", owner=owner, attempt=job["attempt"] + 1, + lease_until=now + self.config.leaseSeconds * 1000, updated_ms=now) + self.db.update(conn, jobs, values, jobs.c.id == job["id"]) + self.db.insert(conn, attempts, dict(job_id=job["id"], number=values["attempt"], + owner=owner, status="running", started_ms=now, finished_ms=None, result_json="{}")) + if not self.db.rows(inbox, inbox.c.event_id == event["id"], connection=conn): + self.db.insert(conn, inbox, dict(event_id=event["id"], job_id=job["id"], + status="received", received_ms=now, finished_ms=None)) + return {**job, **values} + + def _retry_queued(self, conn, job, delay): + new = {**job, "generation": job["generation"] + 1} + self.db.update(conn, jobs, dict(status="queued", generation=new["generation"], owner=None, + lease_until=0, effects_started=0, updated_ms=now_ms(conn)), jobs.c.id == job["id"]) + self._enqueue(conn, new, now_ms(conn) + delay) + + def require_owner(self, conn, identifier, owner): + row = self._row(conn, jobs, identifier, lock=True) + if row["status"] != "running" or row["owner"] != owner or row["lease_until"] < now_ms(conn): + raise LeaseLost("task lease lost; do not repeat external effects") + return row + + def heartbeat(self, identifier, owner): + with self.db.transaction() as conn: + row = self.require_owner(conn, identifier, owner) + self.db.update(conn, jobs, dict(lease_until=now_ms(conn) + self.config.leaseSeconds * 1000, + updated_ms=now_ms(conn)), jobs.c.id == identifier) + return bool(row["cancel_requested"]) + + def begin_effects(self, identifier, owner): + with self.db.transaction() as conn: + row = self.require_owner(conn, identifier, owner) + if row["cancel_requested"]: + raise LeaseLost("task cancellation requested before effects") + self.db.update(conn, jobs, {"effects_started": 1}, jobs.c.id == identifier) + + def finish(self, identifier, owner, status, result, *, followup=None): + if status not in TERMINAL | {"needs-reconciliation"}: + raise ValueError("invalid completion status") + with self.db.transaction() as conn: + job = self.require_owner(conn, identifier, owner) + now = now_ms(conn) + self.db.update(conn, jobs, dict(status=status, result_json=encode(result), + owner=None, lease_until=0, updated_ms=now), jobs.c.id == identifier) + self.db.update(conn, attempts, dict(status=status, result_json=encode(result), finished_ms=now), + attempts.c.job_id == identifier, attempts.c.number == job["attempt"]) + self.db.update(conn, inbox, dict(status="completed", finished_ms=now), inbox.c.job_id == identifier) + self._run_status(conn, job, status, result) + if followup: + self.submit("memory", followup, "memory:" + identifier, connection=conn) + + def _run_status(self, conn, job, status, result): + if not job["run_id"]: + return + mapped = {"succeeded": "completed", "cancelled": "cancelled", "queued": "queued", + "running": "running"}.get(status, "failed") + current = self.db.rows(runs, runs.c.id == job["run_id"], connection=conn, lock=True) + if current and (current[0]["status"] not in {"completed", "failed", "cancelled"} + or current[0]["phase"] == "needs-reconciliation"): + self.db.update(conn, runs, dict(status=mapped, phase=status, result_json=encode(result), updated_at=timestamp()), + runs.c.id == job["run_id"]) + self.platform._add_event(conn, job["run_id"], mapped, + {"job_id": job["id"], "phase": status, "message": result.get("reason", status)}) + + def cancel(self, identifier): + with self.db.transaction() as conn: + job = self._row(conn, jobs, identifier, lock=True) + if job["status"] in TERMINAL: + return self.public(job) + status = "cancelled" if job["status"] == "queued" else job["status"] + self.db.update(conn, jobs, dict(cancel_requested=1, status=status, updated_ms=now_ms(conn)), jobs.c.id == identifier) + if status == "cancelled": + self._run_status(conn, job, status, {"reason": "cancelled before execution"}) + return self.get(identifier) + + def recover(self): + recovered = [] + with self.db.transaction() as conn: + self.db.lock(conn, "task-claim") + now = now_ms(conn) + rows = conn.execute(select(jobs).where(*self.db.predicate(jobs, + jobs.c.status == "running", jobs.c.lease_until < now)) + .limit(50).with_for_update(skip_locked=True)).mappings().all() + for row in rows: + job = dict(row) + status = "needs-reconciliation" if job["effects_started"] and job["kind"] != "memory" else "queued" + persisted = self.db.rows(runs, runs.c.id == job["run_id"], connection=conn) if job["run_id"] else [] + completed = persisted and persisted[0]["status"] in {"completed", "failed", "cancelled"} + if completed: + status = {"completed": "succeeded", "failed": "failed", "cancelled": "cancelled"}[persisted[0]["status"]] + if status == "queued" and job["cancel_requested"]: + status = "cancelled" + if status == "queued" and job["attempt"] >= self.config.maxAttempts: + status = "dead" + reason = {"reason": "worker lease expired", "external_outcome": "unknown" if job["effects_started"] else "not-started"} + if completed: + reason = json.loads(persisted[0]["result_json"]) + self.db.update(conn, attempts, dict(status="interrupted", finished_ms=now, result_json=encode(reason)), + attempts.c.job_id == job["id"], attempts.c.number == job["attempt"]) + if status == "queued": + self._retry_queued(conn, job, min(30000, 1000 * 2 ** job["attempt"])) + else: + self.db.update(conn, jobs, dict(status=status, owner=None, lease_until=0, + result_json=encode(reason), updated_ms=now), jobs.c.id == job["id"]) + self._run_status(conn, job, status, reason) + recovered.append({"job_id": job["id"], "status": status}) + return recovered + + def reconcile(self, identifier, note): + if not note.strip(): + raise ValueError("record how external execution was checked and stopped") + with self.db.transaction() as conn: + job = self._row(conn, jobs, identifier, lock=True) + if job["status"] != "needs-reconciliation": + raise Conflict("only uncertain tasks require manual reconciliation") + result = {"reason": "manually closed without automatic retry", "operator_note": note[:2000], + "verification": "operator-attested-not-machine-verified"} + self.db.update(conn, jobs, dict(status="failed", result_json=encode(result), updated_ms=now_ms(conn)), jobs.c.id == identifier) + self._run_status(conn, job, "failed", result) + return self.get(identifier) + + def detail(self, identifier): + job = self.get(identifier) + job["attempts"] = [{k: v for k, v in r.items() if k not in {"workspace_id", "owner"}} + for r in self.db.rows(attempts, attempts.c.job_id == identifier, order=(attempts.c.number,))] + job["deliveries"] = [{k: r[k] for k in ("id", "generation", "status", "attempts", "last_error", "published_ms")} + for r in self.db.rows(outbox, outbox.c.job_id == identifier, order=(outbox.c.generation,))] + job["artifacts"] = self.db.rows(artifacts, artifacts.c.job_id == identifier) + with self.db.transaction() as conn: + followup = self.find_key(conn, "memory:" + identifier) + job["followups"] = [self.public(followup)] if followup else [] + return job diff --git a/packages/dsh-triton-riscv/python/codex_agent/tests/fixtures/sqlite_v4.sql b/packages/dsh-triton-riscv/python/codex_agent/tests/fixtures/sqlite_v4.sql new file mode 100644 index 0000000..213c9ce --- /dev/null +++ b/packages/dsh-triton-riscv/python/codex_agent/tests/fixtures/sqlite_v4.sql @@ -0,0 +1,101 @@ +-- Frozen SQLite schemas from commit 1a1be6b, used only to test legacy imports. +CREATE TABLE IF NOT EXISTS sessions ( +id TEXT PRIMARY KEY, +title TEXT NOT NULL, +status TEXT NOT NULL, +created_at TEXT NOT NULL, +updated_at TEXT NOT NULL, +pinned INTEGER NOT NULL DEFAULT 0 +); +CREATE TABLE IF NOT EXISTS messages ( +id TEXT PRIMARY KEY, +session_id TEXT NOT NULL REFERENCES sessions(id) ON DELETE CASCADE, +role TEXT NOT NULL, +content TEXT NOT NULL, +metadata_json TEXT NOT NULL, +created_at TEXT NOT NULL +); +CREATE INDEX IF NOT EXISTS idx_messages_session ON messages(session_id, created_at); +CREATE TABLE IF NOT EXISTS runs ( +id TEXT PRIMARY KEY, +session_id TEXT NOT NULL REFERENCES sessions(id) ON DELETE CASCADE, +intent TEXT NOT NULL, +operator TEXT, +status TEXT NOT NULL, +phase TEXT NOT NULL, +request_json TEXT NOT NULL, +result_json TEXT NOT NULL, +created_at TEXT NOT NULL, +updated_at TEXT NOT NULL +); +CREATE INDEX IF NOT EXISTS idx_runs_session ON runs(session_id, created_at); +CREATE TABLE IF NOT EXISTS events ( +id INTEGER PRIMARY KEY AUTOINCREMENT, +run_id TEXT NOT NULL REFERENCES runs(id) ON DELETE CASCADE, +sequence INTEGER NOT NULL, +event_type TEXT NOT NULL, +payload_json TEXT NOT NULL, +created_at TEXT NOT NULL, +UNIQUE(run_id, sequence) +); +CREATE INDEX IF NOT EXISTS idx_events_run ON events(run_id, sequence); +CREATE TABLE IF NOT EXISTS context_checkpoints ( +session_id TEXT PRIMARY KEY REFERENCES sessions(id) ON DELETE CASCADE, +summary TEXT NOT NULL, +through_message_id TEXT, +source_ids_json TEXT NOT NULL, +metrics_json TEXT NOT NULL, +updated_at TEXT NOT NULL +); +CREATE TABLE IF NOT EXISTS metadata ( +key TEXT PRIMARY KEY, +value TEXT NOT NULL +); +CREATE TABLE IF NOT EXISTS memories ( +id INTEGER PRIMARY KEY AUTOINCREMENT, +fingerprint TEXT NOT NULL UNIQUE, +memory_type TEXT NOT NULL, +operator TEXT NOT NULL, +semantics TEXT NOT NULL, +pytorch_reference TEXT NOT NULL, +tl_ops_json TEXT NOT NULL, +failure_stage TEXT, +error_signature TEXT, +environment_json TEXT NOT NULL, +summary TEXT NOT NULL, +searchable_text TEXT NOT NULL, +outcome TEXT NOT NULL, +evidence_json TEXT NOT NULL, +confidence_grade TEXT NOT NULL, +confidence REAL NOT NULL, +source_run TEXT NOT NULL, +test_sha256 TEXT, +active INTEGER NOT NULL DEFAULT 1, +archive_reason TEXT, +created_at TEXT NOT NULL, +updated_at TEXT NOT NULL, +last_used_at TEXT, +useful_count INTEGER NOT NULL DEFAULT 0, +embedding_json TEXT, +embedding_provider TEXT, +embedding_model TEXT, +embedding_dim INTEGER +); +CREATE INDEX IF NOT EXISTS idx_memories_active ON memories(active); +CREATE INDEX IF NOT EXISTS idx_memories_operator ON memories(operator); +CREATE INDEX IF NOT EXISTS idx_memories_stage ON memories(failure_stage); +CREATE INDEX IF NOT EXISTS idx_memories_error ON memories(error_signature); +CREATE TABLE IF NOT EXISTS memory_chunks ( +memory_id INTEGER NOT NULL, +kind TEXT NOT NULL, +position INTEGER NOT NULL, +text TEXT NOT NULL, +source_field TEXT NOT NULL DEFAULT '', +embedding_json TEXT, +embedding_provider TEXT, +embedding_model TEXT, +embedding_dim INTEGER, +PRIMARY KEY (memory_id, kind, position) +); +CREATE INDEX IF NOT EXISTS idx_memory_chunks_memory ON memory_chunks(memory_id); +CREATE TABLE references_catalog (id TEXT PRIMARY KEY, admitted INTEGER NOT NULL, payload TEXT NOT NULL); diff --git a/packages/dsh-triton-riscv/python/codex_agent/tests/test_demo.py b/packages/dsh-triton-riscv/python/codex_agent/tests/test_demo.py index 477533b..5850151 100644 --- a/packages/dsh-triton-riscv/python/codex_agent/tests/test_demo.py +++ b/packages/dsh-triton-riscv/python/codex_agent/tests/test_demo.py @@ -58,7 +58,7 @@ def test_empty_report_does_not_claim_a_historical_success(self) -> None: "memory": { "stats": {"active": 0}, "results": [], - "database": "memory.sqlite3", + "database": "mysql", }, "project_discovery": { "total_targets": 0, diff --git a/packages/dsh-triton-riscv/python/codex_agent/tests/test_diagnostic_memory.py b/packages/dsh-triton-riscv/python/codex_agent/tests/test_diagnostic_memory.py index efeb390..619e2c5 100644 --- a/packages/dsh-triton-riscv/python/codex_agent/tests/test_diagnostic_memory.py +++ b/packages/dsh-triton-riscv/python/codex_agent/tests/test_diagnostic_memory.py @@ -5,7 +5,7 @@ from pathlib import Path from codex_agent.diagnostic_memory import ( - memory_database_path, + memory_workspace, record_from_validation_receipt, remember_validation, retrieve_memories, @@ -76,7 +76,7 @@ def test_memory_write_is_deduplicated_and_auditable(self) -> None: self.assertEqual(first["status"], "recorded") self.assertEqual(second["status"], "deduplicated") self.assertEqual(first["memory_id"], second["memory_id"]) - with MemoryStore(memory_database_path(root)) as store: + with MemoryStore(memory_workspace(root)) as store: stored = store.list() self.assertEqual(len(stored), 1) self.assertEqual(stored[0]["source_run"], "operator-lifecycle:run-failed") @@ -84,7 +84,7 @@ def test_memory_write_is_deduplicated_and_auditable(self) -> None: def test_retrieval_finds_cross_operator_evidence_and_excludes_current_run(self) -> None: with tempfile.TemporaryDirectory() as temporary: root = Path(temporary) - with MemoryStore(memory_database_path(root)) as store: + with MemoryStore(memory_workspace(root)) as store: expected_id, _ = store.add( MemoryRecord( memory_type="successful-repair", diff --git a/packages/dsh-triton-riscv/python/codex_agent/tests/test_mcp_tools.py b/packages/dsh-triton-riscv/python/codex_agent/tests/test_mcp_tools.py index 8745d7e..33d27c1 100644 --- a/packages/dsh-triton-riscv/python/codex_agent/tests/test_mcp_tools.py +++ b/packages/dsh-triton-riscv/python/codex_agent/tests/test_mcp_tools.py @@ -87,6 +87,7 @@ async def test_mcp_lists_lifecycle_and_calls_discover_operator(self) -> None: "execute_validation_job", "get_validation_job", "get_validation_status", + "inspect_queued_task", "evaluate_plugin_fixtures", }, ) diff --git a/packages/dsh-triton-riscv/python/codex_agent/tests/test_memory.py b/packages/dsh-triton-riscv/python/codex_agent/tests/test_memory.py index 0ca6c4c..082bb09 100644 --- a/packages/dsh-triton-riscv/python/codex_agent/tests/test_memory.py +++ b/packages/dsh-triton-riscv/python/codex_agent/tests/test_memory.py @@ -16,6 +16,7 @@ render_memory_context, split_chunk_text, ) +from codex_agent.storage.schema import chunks as chunk_table from codex_agent.memory_chunking import ChunkPolicy, embedding_input from codex_agent.embeddings import OpenAICompatibleEmbeddingProvider @@ -69,7 +70,7 @@ def record(**overrides: object) -> MemoryRecord: class MemoryTests(unittest.TestCase): def test_deduplicates_redacts_and_soft_archives_memory(self) -> None: with tempfile.TemporaryDirectory() as temp_dir: - path = Path(temp_dir) / "memory.sqlite3" + path = Path(temp_dir) / "memory-workspace" with MemoryStore(path) as store: first_id, first_created = store.add( record(summary="Successful repair api_key=do-not-store") @@ -88,7 +89,7 @@ def test_deduplicates_redacts_and_soft_archives_memory(self) -> None: def test_maintenance_archives_superseded_error_evidence(self) -> None: with tempfile.TemporaryDirectory() as temp_dir: - with MemoryStore(Path(temp_dir) / "memory.sqlite3") as store: + with MemoryStore(Path(temp_dir) / "memory-workspace") as store: first_id, _ = store.add( record( memory_type="failure-diagnosis", @@ -118,7 +119,7 @@ def test_maintenance_archives_superseded_error_evidence(self) -> None: def test_structured_and_embedding_retrieval(self) -> None: with tempfile.TemporaryDirectory() as temp_dir: - path = Path(temp_dir) / "memory.sqlite3" + path = Path(temp_dir) / "memory-workspace" with MemoryStore(path, FakeEmbeddingProvider()) as store: expected_id, _ = store.add(record()) store.add( @@ -226,7 +227,7 @@ def test_ingests_verified_run_as_episode_diagnosis_and_repair(self) -> None: ) self.assertTrue(all(item.confidence_grade in {"A", "C"} for item in records)) - with MemoryStore(root / "memory.sqlite3") as store: + with MemoryStore(root / "memory-workspace") as store: first = ingest_results(store, root / "development") second = ingest_results(store, root / "development") self.assertEqual(first["added"], 3) @@ -235,7 +236,7 @@ def test_ingests_verified_run_as_episode_diagnosis_and_repair(self) -> None: def test_embeds_existing_lexical_memories_and_limits_prompt_context(self) -> None: with tempfile.TemporaryDirectory() as temp_dir: - path = Path(temp_dir) / "memory.sqlite3" + path = Path(temp_dir) / "memory-workspace" with MemoryStore(path) as store: memory_id, _ = store.add(record(summary="exponential formulation " * 100)) self.assertEqual(store.stats()["embedded"], 0) @@ -263,7 +264,7 @@ def test_semantic_chunks_keep_case_identity_and_exclude_environment(self) -> Non def test_three_retrieval_modes_require_matching_chunk_embeddings(self) -> None: with tempfile.TemporaryDirectory() as temp_dir: - path = Path(temp_dir) / "memory.sqlite3" + path = Path(temp_dir) / "memory-workspace" query = MemoryQuery( operator="tanh_and_mul", semantics="Compute tanh(x) multiplied by y.", @@ -285,15 +286,15 @@ def test_three_retrieval_modes_require_matching_chunk_embeddings(self) -> None: self.assertEqual(result[0]["retrieval"]["mode"], mode) with self.assertRaises(ValueError): store.retrieve(query, score_mode="fusion", lexical_weight=1.5) - store.connection.execute("DELETE FROM memory_chunks") - store.connection.commit() + with store.db.transaction() as conn: + store.db.delete(conn, chunk_table) with MemoryStore(path, FakeEmbeddingProvider()) as store: self.assertGreater(store.embed_missing()["embedded_chunks"], 0) self.assertEqual(store.retrieve(query, score_mode="embedding")[0]["id"], expected_id) def test_old_relevant_case_is_retrieved_after_1000_newer_records(self) -> None: with tempfile.TemporaryDirectory() as temp_dir: - path = Path(temp_dir) / "memory.sqlite3" + path = Path(temp_dir) / "memory-workspace" with MemoryStore(path) as store: old_id, _ = store.add(record( operator="old_exact_case", @@ -322,14 +323,12 @@ def test_old_relevant_case_is_retrieved_after_1000_newer_records(self) -> None: def test_short_multiline_evidence_stays_intact_with_source(self) -> None: with tempfile.TemporaryDirectory() as temp_dir: - with MemoryStore(Path(temp_dir) / "memory.sqlite3") as store: + with MemoryStore(Path(temp_dir) / "memory-workspace") as store: memory_id, _ = store.add(record(evidence={ "error_excerpt": ["ll.mlir: error: dialect missing\n linalg.generic\n ^"], })) item = store.list()[0] - chunks = store.connection.execute( - "SELECT * FROM memory_chunks WHERE memory_id = ?", (memory_id,) - ).fetchall() + chunks = store.chunk_rows(memory_id) excerpt = next(row for row in chunks if row["source_field"] == "evidence.error_excerpt[0]") self.assertIn("\n linalg.generic\n ^", excerpt["text"]) self.assertEqual(item["evidence"]["error_excerpt"][0], @@ -422,7 +421,7 @@ def test_verified_repair_episode_keeps_failure_patch_and_regression_distinct(sel def test_retrieval_merges_chunks_by_parent_and_context_contains_key_evidence(self) -> None: with tempfile.TemporaryDirectory() as temp_dir: - with MemoryStore(Path(temp_dir) / "memory.sqlite3", FakeEmbeddingProvider()) as store: + with MemoryStore(Path(temp_dir) / "memory-workspace", FakeEmbeddingProvider()) as store: memory_id, _ = store.add(record(evidence={ "error_excerpt": ["dialect missing\n linalg.generic"], "recommended_actions": ["Try lowering to supported operations"], @@ -446,23 +445,9 @@ def test_retrieval_merges_chunks_by_parent_and_context_contains_key_evidence(sel "source=agent-results/development/run-1"): self.assertIn(phrase, context) - def test_schema_upgrade_keeps_parent_id_and_rebuilds_children(self) -> None: - with tempfile.TemporaryDirectory() as temp_dir: - path = Path(temp_dir) / "memory.sqlite3" - with MemoryStore(path) as store: - memory_id, _ = store.add(record()) - store.connection.execute("UPDATE metadata SET value = '3' WHERE key = 'schema_version'") - store.connection.execute("UPDATE memory_chunks SET text = 'old flattened chunk'") - store.connection.commit() - with MemoryStore(path) as store: - self.assertEqual(store.list()[0]["id"], memory_id) - self.assertEqual(store.list()[0]["source_run"], "agent-results/development/run-1") - self.assertFalse(any(row["text"] == "old flattened chunk" for row in - store.connection.execute("SELECT text FROM memory_chunks"))) - def test_reingest_restores_available_line_breaks_without_new_parent(self) -> None: with tempfile.TemporaryDirectory() as temp_dir: - with MemoryStore(Path(temp_dir) / "memory.sqlite3") as store: + with MemoryStore(Path(temp_dir) / "memory-workspace") as store: old_id, _ = store.add(record( summary="failure at line two", evidence={"error_excerpt": ["failure at line two"]}, @@ -475,31 +460,15 @@ def test_reingest_restores_available_line_breaks_without_new_parent(self) -> Non self.assertFalse(created) self.assertEqual(store.list()[0]["summary"], "failure at\nline two") self.assertIn("\nline two", " ".join( - row["text"] for row in store.connection.execute( - "SELECT text FROM memory_chunks WHERE memory_id = ?", (old_id,) - ) + row["text"] for row in store.chunk_rows(old_id) )) - def test_version_two_database_without_chunk_table_migrates_parent(self) -> None: - with tempfile.TemporaryDirectory() as temp_dir: - path = Path(temp_dir) / "memory.sqlite3" - with MemoryStore(path) as store: - memory_id, _ = store.add(record()) - store.connection.execute("DROP TABLE memory_chunks") - store.connection.execute("UPDATE metadata SET value = '2' WHERE key = 'schema_version'") - store.connection.commit() - with MemoryStore(path) as store: - self.assertEqual(store.list()[0]["id"], memory_id) - self.assertGreater(store.connection.execute( - "SELECT COUNT(*) FROM memory_chunks WHERE memory_id = ?", (memory_id,) - ).fetchone()[0], 0) - def test_openai_compatible_embedding_without_tokenizer_fails_closed(self) -> None: provider = OpenAICompatibleEmbeddingProvider( base_url="http://localhost/v1", api_key="fake", model="test", ) with tempfile.TemporaryDirectory() as temp_dir: - with MemoryStore(Path(temp_dir) / "memory.sqlite3", provider) as store: + with MemoryStore(Path(temp_dir) / "memory-workspace", provider) as store: with self.assertRaisesRegex(RuntimeError, "tokenizer"): store.add(record()) diff --git a/packages/dsh-triton-riscv/python/codex_agent/tests/test_memory_context.py b/packages/dsh-triton-riscv/python/codex_agent/tests/test_memory_context.py index d3e50fd..8bcab71 100644 --- a/packages/dsh-triton-riscv/python/codex_agent/tests/test_memory_context.py +++ b/packages/dsh-triton-riscv/python/codex_agent/tests/test_memory_context.py @@ -134,10 +134,10 @@ def test_actual_local_mcp_message_contains_items_and_context(self): async def check(): r, sources = fixture() with tempfile.TemporaryDirectory() as temp: - root = Path(temp); db = root / "memory.sqlite3" - with MemoryStore(db) as store: + root = Path(temp) + with MemoryStore(root) as store: store.add(record_from_validation_receipt(r, sources=sources)) - env = {"TRITON_RISCV_REPO_ROOT":str(root), "TRITON_RISCV_MEMORY_DB":str(db), + env = {"TRITON_RISCV_REPO_ROOT":str(root), "TRITON_RISCV_EMBEDDING_PROVIDER":"none", "TRITON_RISCV_MEMORY_RETRIEVAL_MODE":"legacy", "TRITON_RISCV_MEMORY_CONTEXT_FORMAT":"compact"} with patch.dict(os.environ, env): diff --git a/packages/dsh-triton-riscv/python/codex_agent/tests/test_memory_evidence.py b/packages/dsh-triton-riscv/python/codex_agent/tests/test_memory_evidence.py index 44b37b1..c984928 100644 --- a/packages/dsh-triton-riscv/python/codex_agent/tests/test_memory_evidence.py +++ b/packages/dsh-triton-riscv/python/codex_agent/tests/test_memory_evidence.py @@ -83,7 +83,7 @@ def test_duplicate_and_changed_source_replace_chunks_without_cross_run_merge(sel changed=json.loads(s[1].content); changed["diff"]="@@ -1 +1 @@\n-old\n+newer" updated=record_from_validation_receipt(r,sources=[s[0],EvidenceSource(s[1].path,json.dumps(changed))]) self.assertEqual(store.add(updated),(mid,False)) - chunks=[dict(c) for c in store.connection.execute("SELECT * FROM memory_chunks")] + chunks=store.chunk_rows() self.assertFalse(any("+new\n" in c["text"] for c in chunks)) r2={**r,"run_id":"run-b"} store.add(record_from_validation_receipt(r2,sources=[EvidenceSource("b.json",json.dumps(r2))])) @@ -109,10 +109,10 @@ def test_real_public_model_and_context_retain_binding(self): from codex_agent.operator_lifecycle import MemoryRetrievalToolResult r,s=fixture() with tempfile.TemporaryDirectory() as temp: - db=Path(temp)/"m.db" - with MemoryStore(db) as store: + root = Path(temp) + with MemoryStore(root) as store: store.add(record_from_validation_receipt(r,sources=s)) - with patch.dict("os.environ", {"TRITON_RISCV_MEMORY_DB":str(db),"TRITON_RISCV_EMBEDDING_PROVIDER":"none","TRITON_RISCV_MEMORY_RETRIEVAL_MODE":"legacy"}): + with patch.dict("os.environ", {"TRITON_RISCV_EMBEDDING_PROVIDER":"none","TRITON_RISCV_MEMORY_RETRIEVAL_MODE":"legacy"}): result=retrieve_memories(Path(temp),operator_name="demo",failure_stage="mlir-translate") typed=MemoryRetrievalToolResult.model_validate(result).model_dump() self.assertTrue(typed["items"][0]["evidence_chain"]["items"]) @@ -126,7 +126,7 @@ def test_missing_bad_sources_safe_and_path_escape_not_read(self): self.assertEqual(len(sources),1) chain=assemble_receipt_evidence(r,sources+[EvidenceSource("broken.json","not json")]) self.assertTrue(any(g["reason"]=="referenced-log-unavailable" for g in chain["gaps"])) - with patch.dict("os.environ", {"TRITON_RISCV_MEMORY_DB":str(Path(temp)/"m.db"),"TRITON_RISCV_EMBEDDING_PROVIDER":"none"}): + with patch.dict("os.environ", {"TRITON_RISCV_EMBEDDING_PROVIDER":"none"}): self.assertEqual(remember_validation(Path(temp),r)["status"],"recorded") def test_public_second_cut_marks_patch_and_bad_confidence_degrades(self): diff --git a/packages/dsh-triton-riscv/python/codex_agent/tests/test_memory_selection.py b/packages/dsh-triton-riscv/python/codex_agent/tests/test_memory_selection.py index 25a64ba..4d879cb 100644 --- a/packages/dsh-triton-riscv/python/codex_agent/tests/test_memory_selection.py +++ b/packages/dsh-triton-riscv/python/codex_agent/tests/test_memory_selection.py @@ -109,11 +109,11 @@ def test_actual_mcp_selection_parameter_default_and_opt_in(self): from codex_agent.harness.mcp_server import server async def check(): with tempfile.TemporaryDirectory() as temp: - root = Path(temp); db = root/'memory.sqlite3' - with MemoryStore(db) as store: + root = Path(temp) + with MemoryStore(root) as store: for i in range(6): store.add(stored(i, operator='alpha')) beta, _ = store.add(stored('beta', operator='beta')) - env = {'TRITON_RISCV_REPO_ROOT':str(root), 'TRITON_RISCV_MEMORY_DB':str(db), + env = {'TRITON_RISCV_REPO_ROOT':str(root), 'TRITON_RISCV_EMBEDDING_PROVIDER':'none', 'TRITON_RISCV_MEMORY_RETRIEVAL_MODE':'legacy'} with patch.dict(os.environ, env): os.environ.pop('TRITON_RISCV_MEMORY_CONTEXT_FORMAT', None) diff --git a/packages/dsh-triton-riscv/python/codex_agent/tests/test_migrate_memory.py b/packages/dsh-triton-riscv/python/codex_agent/tests/test_migrate_memory.py deleted file mode 100644 index 490b111..0000000 --- a/packages/dsh-triton-riscv/python/codex_agent/tests/test_migrate_memory.py +++ /dev/null @@ -1,48 +0,0 @@ -from pathlib import Path -import hashlib -import json -import sqlite3 -import tempfile -import unittest - -from codex_agent.memory import MemoryRecord, MemoryStore -from codex_agent.migrate_memory import migrate_memory - - -class MemoryMigrationTests(unittest.TestCase): - def test_backup_keeps_original_and_evidence_without_inventing_labels(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) / "old" - run = root / "agent-results/development/example" - run.mkdir(parents=True) - log = run / "validation.log" - log.write_text("1 failed\n") - (run / ".env").write_text("DO_NOT_COPY=this\n") - db = root / "agent-results/memory.sqlite3" - with MemoryStore(db) as store: - store.add(MemoryRecord(memory_type="failure-diagnosis", operator="demo", semantics="square", - pytorch_reference="x*x", summary="Failure, cause unknown", outcome="failed", confidence_grade="C", - source_run="agent-results/development/example", evidence={"log_path": str(log), - "patch_path": "agent-results/missing.diff", "recommended_actions": ["inspect source"]})) - before = hashlib.sha256(db.read_bytes()).hexdigest() - out = Path(temporary) / "new" - result = migrate_memory(db, root, out) - self.assertTrue(result["source_file_unchanged"]) - self.assertEqual(before, hashlib.sha256(db.read_bytes()).hexdigest()) - self.assertEqual(len(result["records"]), 1) - self.assertEqual(len(result["gaps"]), 1) - with sqlite3.connect(out / "memory.sqlite3") as con: - row = con.execute("SELECT outcome,evidence_json,source_run FROM memories").fetchone() - self.assertEqual(row[0], "failed") - evidence = json.loads(row[1]) - self.assertEqual(Path(evidence["log_path"]).read_text(), "1 failed\n") - self.assertEqual(evidence["recommended_actions"], ["inspect source"]) - self.assertNotIn("applied_action", evidence) - self.assertTrue(Path(row[2]).is_dir()) - self.assertFalse(list(out.rglob(".env"))) - with self.assertRaises(FileExistsError): - migrate_memory(db, root, out) - - -if __name__ == "__main__": - unittest.main() diff --git a/packages/dsh-triton-riscv/python/codex_agent/tests/test_mysql_storage.py b/packages/dsh-triton-riscv/python/codex_agent/tests/test_mysql_storage.py new file mode 100644 index 0000000..1576055 --- /dev/null +++ b/packages/dsh-triton-riscv/python/codex_agent/tests/test_mysql_storage.py @@ -0,0 +1,270 @@ +"""MySQL integration tests. Use a disposable schema, never a production database.""" +from concurrent.futures import ThreadPoolExecutor +from contextlib import closing +import hashlib +import json +import os +from pathlib import Path +import sqlite3 +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import patch + +from sqlalchemy import text +from sqlalchemy.exc import TimeoutError as PoolTimeout + +from codex_agent.memory import MemoryStore +from codex_agent.platform.store import PlatformStore +from codex_agent.process_control import validation_environment +from codex_agent.storage.database import StorageError, WorkspaceDatabase, _engine +from codex_agent.storage.import_sqlite import import_sqlite, snapshot +from codex_agent.storage.schema import memories, references, sessions, workspaces +from codex_agent.tests.test_memory import FakeEmbeddingProvider, record + + +def insert_legacy(conn, table, row): + names = ", ".join('"' + name + '"' for name in row) + conn.execute('INSERT INTO "' + table + '" (' + names + ') VALUES (' + + ",".join("?" for _ in row) + ")", tuple(row.values())) + + +class MySQLStorageTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + + def legacy_source(self, *, old_schema=False): + path = self.root / "legacy.sqlite3" + conn = sqlite3.connect(path) + self.addCleanup(conn.close) + conn.executescript((Path(__file__).parent / "fixtures/sqlite_v4.sql").read_text()) + conn.execute("INSERT INTO metadata VALUES ('schema_version', ?)", ("2" if old_schema else "4",)) + now = "2026-09-01T00:00:00Z" + insert_legacy(conn, "sessions", dict(id="session", title="legacy", status="active", + created_at=now, updated_at=now, pinned=1)) + for identifier in ("z-first", "a-second"): + insert_legacy(conn, "messages", dict(id=identifier, session_id="session", role="user", + content="unchanged " + identifier, metadata_json='{"kind":"fixture"}', created_at=now)) + insert_legacy(conn, "runs", dict(id="run", session_id="session", intent="validate", operator="demo", + status="failed", phase="completed", request_json="{}", result_json='{"exit_code":1}', + created_at=now, updated_at=now)) + insert_legacy(conn, "events", dict(id=42, run_id="run", sequence=7, event_type="completed", + payload_json='{"status":"failed"}', created_at=now)) + insert_legacy(conn, "context_checkpoints", dict(session_id="session", summary="failure not repaired", + through_message_id="a-second", source_ids_json='["run"]', metrics_json="{}", updated_at=now)) + insert_legacy(conn, "references_catalog", dict(id="reference", admitted=1, + payload='{"source":"docs/rule.md"}')) + source = MemoryStore(self.root / "fixture-seed") + source.add(record(outcome="failed", confidence_grade="C", evidence={ + "error_excerpt": ["linalg error"], + "recommended_actions": ["try supported primitives"], + })) + for row in source.db.rows(memories): + row.pop("workspace_id") + row["id"] = 17 + insert_legacy(conn, "memories", row) + for row in source.chunk_rows(): + row.pop("workspace_id") + row["memory_id"] = 17 + insert_legacy(conn, "memory_chunks", row) + if old_schema: + conn.execute("DROP TABLE memory_chunks") + conn.execute("ALTER TABLE memories DROP COLUMN embedding_json") + conn.commit() + return path + + def test_real_mysql_and_workspace_isolation(self): + left, right = MemoryStore(self.root / "left"), MemoryStore(self.root / "right") + with left.db.transaction() as conn: + self.assertTrue(conn.execute(text("SELECT VERSION()")).scalar().startswith("8.")) + self.assertEqual(left.add(record())[0], 1) + self.assertEqual(right.add(record(operator="other"))[0], 1) + left.archive(1, "fixture") + self.assertEqual(left.list(), []) + self.assertEqual(right.list()[0]["operator"], "other") + + def test_concurrent_duplicate_ingest_has_one_parent_and_complete_chunks(self): + root = self.root / "concurrent-memory" + with ThreadPoolExecutor(max_workers=6) as pool: + results = list(pool.map(lambda _: MemoryStore(root).add(record()), range(24))) + store = MemoryStore(root) + self.assertEqual(sum(created for _, created in results), 1) + self.assertEqual(len(store.list()), 1) + self.assertGreater(len(store.chunk_rows()), 0) + self.assertEqual({identifier for identifier, _ in results}, {1}) + + def test_identical_session_id_in_other_workspace_is_not_visible(self): + left, right = PlatformStore(self.root / "one"), PlatformStore(self.root / "two") + original = left.create_session("one") + with right.db.transaction() as conn: + right.db.insert(conn, sessions, {**original, "title": "two"}) + self.assertEqual(left.get_session(original["id"])["title"], "one") + self.assertEqual(right.get_session(original["id"])["title"], "two") + left.delete_session(original["id"]) + self.assertEqual(right.get_session(original["id"])["title"], "two") + + def test_event_serialization_across_independent_processes(self): + root = self.root / "processes" + store = PlatformStore(root) + run = store.create_run(store.create_session()["id"], "validate", "demo", {}) + code = ("from codex_agent.platform.store import PlatformStore; from pathlib import Path; " + "import sys; s=PlatformStore(Path(sys.argv[1])); " + "[s.add_event(sys.argv[2], 'output', {'n':n}) for n in range(10)]") + def worker(_): + subprocess.run([sys.executable, "-I", "-c", code, str(root), run["id"]], + check=True, capture_output=True, text=True, timeout=30) + with ThreadPoolExecutor(max_workers=4) as pool: + list(pool.map(worker, range(4))) + self.assertEqual([r["sequence"] for r in store.list_events(run["id"])], list(range(41))) + + def test_connection_pool_has_a_finite_wait_and_no_unbounded_overflow(self): + engine = _engine(os.environ["TRITON_MYSQL_URL"], 1, 0, 1, os.getpid()) + try: + with engine.connect(): + with self.assertRaises(PoolTimeout): + engine.connect() + self.assertEqual(engine.pool.checkedout(), 0) + finally: + engine.dispose() + + def test_concurrent_event_sequences_are_unique_and_resumable(self): + root = self.root / "events" + store = PlatformStore(root) + session = store.create_session() + run = store.create_run(session["id"], "validate", "demo", {}) + with ThreadPoolExecutor(max_workers=6) as pool: + list(pool.map(lambda n: PlatformStore(root).add_event(run["id"], "output", {"n": n}), range(40))) + events = PlatformStore(root).list_events(run["id"]) + self.assertEqual([row["sequence"] for row in events], list(range(41))) + self.assertEqual(len({row["id"] for row in events}), 41) + self.assertEqual(PlatformStore(root).list_events(run["id"], after=39), [events[-1]]) + + def test_failed_parent_chunk_update_rolls_back(self): + store = MemoryStore(self.root / "atomic") + store.add(record()) + before, before_chunks = store.list(), store.chunk_rows() + with patch.object(store, "_insert_chunks", side_effect=RuntimeError("injected disk failure")): + with self.assertRaisesRegex(RuntimeError, "injected"): + store.add(record(evidence={"error_excerpt": ["changed evidence"]})) + self.assertEqual(store.list(), before) + self.assertEqual(store.chunk_rows(), before_chunks) + + def test_reingestion_without_provider_keeps_existing_vectors(self): + root = self.root / "vectors" + store = MemoryStore(root, FakeEmbeddingProvider()) + store.add(record()) + before = store.chunk_rows() + MemoryStore(root).add(record()) + self.assertEqual(MemoryStore(root).chunk_rows(), before) + self.assertTrue(MemoryStore(root).list()[0]["embedding"]) + with patch.object(store.embedding_provider, "embed", wraps=store.embedding_provider.embed) as embed: + store.add(record()) + embed.assert_not_called() + store.add(record(confidence_grade="D")) + self.assertEqual(store.list()[0]["confidence_grade"], "A") + + def test_active_run_blocks_delete_and_finished_delete_cascades(self): + store = PlatformStore(self.root / "deletion") + session = store.create_session() + store.add_message(session["id"], "user", "test") + run = store.create_run(session["id"], "validate", "demo", {}, status="running") + store.save_context_checkpoint(session["id"], summary="test", through_message_id=None, + source_ids=[], metrics={}) + with self.assertRaises(ValueError): + store.delete_session(session["id"]) + store.update_run(run["id"], status="completed") + store.delete_session(session["id"]) + self.assertEqual(store.list_messages(session["id"]), []) + self.assertEqual(store.list_events(run["id"]), []) + self.assertIsNone(store.get_context_checkpoint(session["id"])) + + def test_readonly_import_preserves_ids_evidence_order_and_counters(self): + source = self.legacy_source() + before = hashlib.sha256(source.read_bytes()).hexdigest() + target = self.root / "imported" + receipt = import_sqlite(target, platform=source, memory=source, references=source) + self.assertEqual(hashlib.sha256(source.read_bytes()).hexdigest(), before) + self.assertFalse(receipt["labels_reaudited"]) + self.assertFalse(receipt["evidence_paths_rewritten"]) + platform = PlatformStore(target) + self.assertEqual([r["id"] for r in platform.list_messages("session")], ["z-first", "a-second"]) + self.assertEqual(platform.get_context_checkpoint("session")["summary"], "failure not repaired") + event = platform.add_event("run", "output", {}) + self.assertEqual((event["id"], event["sequence"]), (43, 8)) + platform.add_message("session", "user", "third") + self.assertEqual(platform.list_messages("session")[-1]["content"], "third") + memories_store = MemoryStore(target) + item = memories_store.list()[0] + self.assertEqual(item["id"], 17) + self.assertEqual(item["outcome"], "failed") + self.assertEqual(item["confidence_grade"], "C") + self.assertEqual(item["evidence"]["recommended_actions"], ["try supported primitives"]) + self.assertNotIn("applied_action", item["evidence"]) + self.assertEqual(memories_store.add(record(operator="new"))[0], 18) + self.assertEqual(len(memories_store.db.rows(references)), 1) + with self.assertRaisesRegex(ValueError, "already exists"): + import_sqlite(target, memory=source) + + def test_old_schema_rebuilds_chunks_without_fabricating_results(self): + source = self.legacy_source(old_schema=True) + target = self.root / "old" + result = import_sqlite(target, memory=source) + self.assertEqual(result["old_memory_version"], 2) + store = MemoryStore(target) + self.assertGreater(len(store.chunk_rows(17)), 0) + self.assertEqual(store.list()[0]["embedding"], []) + self.assertEqual(store.list()[0]["outcome"], "failed") + self.assertNotIn("Validation result", " ".join(r["text"] for r in store.chunk_rows())) + + def test_import_foreign_key_failure_rolls_back_entire_workspace(self): + source = self.legacy_source() + with closing(sqlite3.connect(source)) as conn, conn: + conn.execute("UPDATE messages SET session_id = 'orphan'") + target = self.root / "broken" + with self.assertRaises(StorageError): + import_sqlite(target, platform=source, memory=source) + db = WorkspaceDatabase(target) + self.assertEqual(db.rows(workspaces), []) + self.assertEqual(db.rows(memories), []) + with closing(sqlite3.connect(source)) as conn, conn: + conn.execute("UPDATE messages SET session_id = 'session'") + self.assertEqual(import_sqlite(target, platform=source)["counts"]["sessions"], 1) + + def test_sqlite_snapshot_is_readonly(self): + source = self.legacy_source() + with snapshot(source) as conn: + with self.assertRaises(sqlite3.OperationalError): + conn.execute("DELETE FROM memories") + + def test_readonly_import_includes_committed_wal_records(self): + source = self.legacy_source() + with closing(sqlite3.connect(source)) as conn, conn: + conn.execute("PRAGMA journal_mode=WAL") + conn.execute("UPDATE sessions SET title='committed WAL'") + conn.commit() + self.assertTrue(Path(str(source) + "-wal").exists()) + target = self.root / "wal" + import_sqlite(target, platform=source) + self.assertEqual(PlatformStore(target).get_session("session")["title"], "committed WAL") + + def test_missing_configuration_fails_without_creating_sqlite_files(self): + with patch.dict(os.environ, {}, clear=True): + with self.assertRaisesRegex(StorageError, "TRITON_MYSQL_URL"): + MemoryStore(self.root) + self.assertEqual(list(self.root.iterdir()), []) + + def test_database_credentials_are_not_forwarded_to_validation(self): + with patch.dict(os.environ, {"TRITON_MYSQL_URL": "mysql+pymysql://user:secret@localhost/db"}): + self.assertNotIn("TRITON_MYSQL_URL", validation_environment()) + + def test_invalid_url_errors_do_not_expose_secret(self): + with self.assertRaises(StorageError) as caught: + _engine("postgresql://user:secret@localhost/db", 1, 0, 1, os.getpid()) + self.assertNotIn("secret", str(caught.exception)) + + +if __name__ == "__main__": + unittest.main() diff --git a/packages/dsh-triton-riscv/python/codex_agent/tests/test_platform_service.py b/packages/dsh-triton-riscv/python/codex_agent/tests/test_platform_service.py index 1c6d8b5..91fa992 100644 --- a/packages/dsh-triton-riscv/python/codex_agent/tests/test_platform_service.py +++ b/packages/dsh-triton-riscv/python/codex_agent/tests/test_platform_service.py @@ -254,7 +254,7 @@ def make_components( }, ) agent = HarnessAgent(settings, FakeHarnessBackend()) - store = PlatformStore(results / "platform.sqlite3") + store = PlatformStore(root) service = PlatformService(root, store, agent) executor = HarnessRunExecutor(root, store, agent, workers=1) return store, service, executor @@ -327,7 +327,7 @@ def test_harness_error_never_persists_an_empty_assistant_message(self) -> None: results.mkdir() settings = HarnessSettings.from_env(root, {"ISRC_API_KEY": "test-key"}) agent = HarnessAgent(settings, ErrorHarnessBackend()) - store = PlatformStore(results / "platform.sqlite3") + store = PlatformStore(root) service = PlatformService(root, store, agent) executor = HarnessRunExecutor(root, store, agent, workers=1) self.addCleanup(executor.close) diff --git a/packages/dsh-triton-riscv/python/codex_agent/tests/test_platform_store.py b/packages/dsh-triton-riscv/python/codex_agent/tests/test_platform_store.py index f6c59b9..4caba5d 100644 --- a/packages/dsh-triton-riscv/python/codex_agent/tests/test_platform_store.py +++ b/packages/dsh-triton-riscv/python/codex_agent/tests/test_platform_store.py @@ -10,7 +10,7 @@ class PlatformStoreTests(unittest.TestCase): def test_persists_session_messages_runs_and_events(self) -> None: with tempfile.TemporaryDirectory() as temp_dir: - store = PlatformStore(Path(temp_dir) / "platform.sqlite3") + store = PlatformStore(Path(temp_dir)) session = store.create_session() message = store.add_message( session["id"], "user", "验证 relu", {"source": "test"} @@ -36,7 +36,7 @@ def test_persists_session_messages_runs_and_events(self) -> None: def test_rejects_unknown_session(self) -> None: with tempfile.TemporaryDirectory() as temp_dir: - store = PlatformStore(Path(temp_dir) / "platform.sqlite3") + store = PlatformStore(Path(temp_dir)) with self.assertRaises(KeyError): store.get_session("missing") diff --git a/packages/dsh-triton-riscv/python/codex_agent/tests/test_plugin_migration.py b/packages/dsh-triton-riscv/python/codex_agent/tests/test_plugin_migration.py index cf7acdd..2ff0268 100644 --- a/packages/dsh-triton-riscv/python/codex_agent/tests/test_plugin_migration.py +++ b/packages/dsh-triton-riscv/python/codex_agent/tests/test_plugin_migration.py @@ -11,8 +11,7 @@ from fastapi.testclient import TestClient from mcp import Client -import codex_agent -from codex_agent.diagnostic_memory import memory_database_path, remember_validation +from codex_agent.diagnostic_memory import memory_workspace, remember_validation from codex_agent.harness import HarnessAgent, HarnessSettings from codex_agent.harness.mcp_server import server from codex_agent.platform.api import create_app @@ -26,15 +25,22 @@ class InstalledMigrationTests(unittest.TestCase): def test_installed_package_cannot_be_shadowed_by_target_checkout(self): with tempfile.TemporaryDirectory() as temporary: root = Path(temporary) + command = [sys.executable, "-I", "-c", "import codex_agent; print(codex_agent.__file__)"] + # The test runner may import source; isolated children use the installed package. + baseline = subprocess.run(command, cwd=root, capture_output=True, text=True, timeout=20) + self.assertEqual(baseline.returncode, 0, baseline.stderr) + installed = Path(baseline.stdout.strip()).resolve() + self.assertTrue(installed.is_file()) shadow = root / "codex_agent" shadow.mkdir() (shadow / "__init__.py").write_text("raise RuntimeError('old checkout imported')") completed = subprocess.run( - [sys.executable, "-I", "-c", "import codex_agent; print(codex_agent.__file__)"], + command, env={**os.environ, "PYTHONPATH": str(root)}, cwd=root, capture_output=True, text=True, timeout=20, ) self.assertEqual(completed.returncode, 0, completed.stderr) - self.assertEqual(Path(completed.stdout.strip()).resolve(), Path(codex_agent.__file__).resolve()) + self.assertEqual(Path(completed.stdout.strip()).resolve(), installed) + self.assertNotEqual(installed, (shadow / "__init__.py").resolve()) def test_ui_and_state_do_not_require_backend_inside_target_checkout(self): with tempfile.TemporaryDirectory() as temporary: @@ -52,14 +58,14 @@ def test_ui_and_state_do_not_require_backend_inside_target_checkout(self): self.assertEqual(client.get("/api/health").json()["repo_root"], str(root.resolve())) session = client.post("/api/sessions", json={"title": "migration"}).json() self.assertEqual(client.delete(f"/api/sessions/{session['id']}").status_code, 204) - self.assertTrue((state / "platform.sqlite3").is_file()) + self.assertFalse((state / "platform.sqlite3").exists()) self.assertFalse((root / "codex_agent").exists()) self.assertFalse((root / "agent-results/platform.sqlite3").exists()) def test_managed_context_reaches_executor_and_persists(self): with tempfile.TemporaryDirectory() as temporary: root = Path(temporary) - store = PlatformStore(root / "state.sqlite3") + store = PlatformStore(root) settings = HarnessSettings.from_env(root, {"TRITON_RISCV_CONTEXT_SUMMARY_MODE": "extractive"}) executor = HarnessRunExecutor(root, store, HarnessAgent(settings, FakeHarnessBackend())) try: @@ -80,7 +86,6 @@ async def test_mcp_retrieves_existing_evidence_without_inventing_repair(self): with tempfile.TemporaryDirectory() as temporary: root = Path(temporary) with patch.dict(os.environ, {"TRITON_RISCV_REPO_ROOT": str(root), - "TRITON_RISCV_MEMORY_DB": str(root / "history.sqlite3"), "TRITON_RISCV_EMBEDDING_PROVIDER": "none", "TRITON_RISCV_MEMORY_RETRIEVAL_MODE": "legacy"}): remember_validation(root, failed_receipt("previous"), OPERATOR) @@ -96,4 +101,5 @@ async def test_mcp_retrieves_existing_evidence_without_inventing_repair(self): self.assertEqual(item["recommended_actions"], ["Preserve the failing IR."]) self.assertIsNone(item["applied_action"]) self.assertEqual(item["source_run"], "operator-lifecycle:previous") - self.assertEqual(Path(payload["database"]), memory_database_path(root)) + self.assertEqual(payload["database"], "mysql") + self.assertEqual(Path(payload["workspace"]), memory_workspace(root)) diff --git a/packages/dsh-triton-riscv/python/codex_agent/tests/test_redis_cache.py b/packages/dsh-triton-riscv/python/codex_agent/tests/test_redis_cache.py new file mode 100644 index 0000000..0260626 --- /dev/null +++ b/packages/dsh-triton-riscv/python/codex_agent/tests/test_redis_cache.py @@ -0,0 +1,369 @@ +"""Redis/MySQL integration tests; only disposable services may be supplied.""" +from concurrent.futures import ThreadPoolExecutor +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import threading +import time +import unittest +from unittest.mock import patch + +import redis +from sqlalchemy import select + +from codex_agent.memory import MemoryQuery, MemoryStore +from codex_agent.process_control import validation_environment +from codex_agent.runtime_config import CONFIG_ENV, runtime_config +from codex_agent.storage.cache import CacheBusy, RELEASE, VersionChanged +from codex_agent.storage.schema import memories +from codex_agent.tests.test_memory import record + + +@unittest.skipUnless(os.environ.get("TRITON_TEST_REDIS_URL") and os.environ.get("TRITON_MYSQL_URL"), + "disposable Redis and MySQL required") +class RedisCacheTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + with redis.Redis.from_url(os.environ["TRITON_TEST_REDIS_URL"], + socket_timeout=2, socket_connect_timeout=2, + decode_responses=True) as client: + settings = client.config_get("maxmemory*") + if int(settings.get("maxmemory", 0)) <= 0 or settings.get("maxmemory-policy") != "noeviction": + raise RuntimeError("Disposable Redis requires maxmemory > 0 and maxmemory-policy noeviction; " + "configure the test service before running cache tests") + + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.cfg = {"enabled": True, "requestsPerSecond": 10000, "rebuildsPerSecond": 1000} + self.env = patch.dict(os.environ, {"TRITON_REDIS_URL": os.environ["TRITON_TEST_REDIS_URL"], + CONFIG_ENV: self.document()}) + self.env.start() + self.addCleanup(self.env.stop) + self.store = MemoryStore(self.root) + self.backend = self.store.cache.backend + self.backend.retry_at = 0 + self.backend.fallback_times = [] + self.client = self.backend.client + self.query = MemoryQuery("relu_and_mul", "relu multiply", "torch.relu(x) * y") + + def document(self): + return json.dumps({"schemaVersion": 1, "repoRoot": str(self.root), + "stateDir": str(self.root / 'state'), "cache": self.cfg}) + + def reconfigure(self, **kwargs): + self.cfg.update(kwargs) + os.environ[CONFIG_ENV] = self.document() + self.store = MemoryStore(self.root) + return self.store + + def test_same_query_reuses_ranking_and_keeps_usage_update(self): + self.store.add(record(operator="relu_and_mul")) + with patch.object(self.store, "rank_candidates", wraps=self.store.rank_candidates) as rank: + first, second = self.store.retrieve(self.query), self.store.retrieve(self.query) + self.assertEqual(first, second) + self.assertEqual(rank.call_count, 1) + self.assertIsNotNone(self.store.db.rows(memories)[0]["last_used_at"]) + + def test_negative_cache_and_creation_invalidation(self): + with patch.object(self.store.db, "rows", wraps=self.store.db.rows) as rows: + self.assertIsNone(self.store.get(1)) + self.assertIsNone(self.store.get(1)) + self.assertEqual(sum(c.args[0] is memories for c in rows.call_args_list), 1) + identifier, _ = self.store.add(record()) + self.assertEqual(self.store.get(identifier)["id"], identifier) + + def test_archive_stats_and_write_invalidation(self): + identifier, _ = self.store.add(record()) + self.assertEqual(self.store.stats()["active"], 1) + self.assertIsNotNone(self.store.get(identifier)) + self.store.archive(identifier, "test") + self.assertIsNone(self.store.get(identifier)) + self.assertEqual(self.store.stats()["active"], 0) + + def test_failed_write_does_not_publish_revision(self): + self.store.add(record()) + before = self.store.db.revision("memory") + with self.assertRaises(RuntimeError): + with self.store.db.transaction() as conn: + self.store.db.update(conn, memories, {"summary": "not committed"}) + raise RuntimeError("rollback") + self.assertEqual(self.store.db.revision("memory"), before) + self.assertNotEqual(self.store.get(1)["summary"], "not committed") + + def test_query_environment_mode_limit_and_exclusions_do_not_share_cache(self): + self.store.add(record(operator="relu_and_mul")) + with patch.object(self.store, "rank_candidates", wraps=self.store.rank_candidates) as rank: + self.store.retrieve(self.query) + self.store.retrieve(self.query, limit=1) + self.store.retrieve(self.query, score_mode="jaccard") + self.store.retrieve(self.query, exclude_source_runs=["different-run"]) + self.store.retrieve(MemoryQuery("relu_and_mul", "relu multiply", "torch.relu(x) * y", + environment={"triton": "different-version"})) + self.assertEqual(rank.call_count, 5) + + def test_workspace_isolation(self): + self.store.add(record()) + other = MemoryStore(self.root / "other") + self.assertIsNotNone(self.store.get(1)) + self.assertIsNone(other.get(1)) + + def test_shared_tool_retrieval_path_uses_cache_without_changing_context(self): + from codex_agent.diagnostic_memory import retrieve_memories + self.store.add(record(operator='relu_and_mul')) + original = MemoryStore.rank_candidates + with patch.object(MemoryStore, 'rank_candidates', autospec=True, side_effect=original) as rank: + first = retrieve_memories(self.root, operator_name='relu_and_mul') + second = retrieve_memories(self.root, operator_name='relu_and_mul') + self.assertEqual(first['status'], 'found') + self.assertEqual(first, second) + self.assertEqual(rank.call_count, 1) + + def test_cached_reference_still_rechecks_source_evidence(self): + from codex_agent.tests import test_reference_library as fixtures + fixture = fixtures.ReferenceLibraryTests() + fixture.setUp() + self.addCleanup(fixture.doCleanups) + fixture.build() + self.assertEqual(len(fixtures.search_library(fixture.output, 'demo', environment=fixtures.ENV)['items']), 1) + fixture.impl.write_text(fixtures.IMPLEMENTATION + '# changed\n') + result = fixtures.search_library(fixture.output, 'demo', environment=fixtures.ENV) + self.assertEqual(result['items'], []) + self.assertEqual(result['rejected'][0]['reason'], 'source-or-evidence-changed') + + def test_same_key_concurrency_has_one_rebuild(self): + barrier = threading.Barrier(12) + count = 0 + def load(): + nonlocal count + count += 1 + time.sleep(0.15) + return {"ok": True} + def request(_): + barrier.wait() + return self.store.cache.get("concurrency", {}, load) + with ThreadPoolExecutor(max_workers=12) as pool: + results = list(pool.map(request, range(12))) + self.assertEqual(count, 1) + self.assertTrue(all(r == {"ok": True} for r in results)) + + def test_lease_expiry_cannot_publish_or_delete_successors_lock(self): + self.reconfigure(lockMs=100) + entered = threading.Event() + def slow(): + entered.set() + time.sleep(0.3) + return "old" + with ThreadPoolExecutor(max_workers=2) as pool: + old = pool.submit(self.store.cache.get, "lease", {}, slow) + entered.wait(2) + time.sleep(0.15) + self.assertEqual(self.store.cache.get("lease", {}, lambda: "new"), "new") + with self.assertRaises(CacheBusy): + old.result() + self.assertEqual(self.store.cache.get("lease", {}, lambda: "wrong"), "new") + key = self.store.cache.prefix + ':owner-test' + self.client.set(key, 'successor', px=1000) + self.assertEqual(self.client.eval(RELEASE, 1, key, 'predecessor'), 0) + self.assertEqual(self.client.get(key), b'successor') + + def test_crashed_process_lock_expires(self): + cache = self.store.cache + key = cache.key("crash", {}, self.store.db.revision("memory")) + ':lock' + code = "import os,redis,sys; r=redis.Redis.from_url(os.environ['TRITON_REDIS_URL']); r.set(sys.argv[1],'crashed',px=150); os._exit(0)" + subprocess.run([sys.executable, '-I', '-c', code, key], check=True, timeout=10) + self.assertEqual(cache.get("crash", {}, lambda: "recovered"), "recovered") + + def test_concurrent_write_during_load_retries_new_version(self): + self.store.add(record()) + count = 0 + def load(): + nonlocal count + count += 1 + old = self.store.db.rows(memories)[0]["summary"] + if count == 1: + with ThreadPoolExecutor(max_workers=1) as pool: + pool.submit(self.store.archive, 1, "changed").result() + return old if count == 1 else "new" + self.assertEqual(self.store.cache.get("changed", {}, load), "new") + self.assertEqual(count, 2) + + def test_error_is_not_negative_cached(self): + with self.assertRaisesRegex(RuntimeError, "database fixture"): + self.store.cache.get("error", {}, lambda: (_ for _ in ()).throw(RuntimeError("database fixture")), exact_id=1) + self.assertEqual(self.store.cache.get("error", {}, lambda: {"exists": True}, exact_id=1), {"exists": True}) + + def test_empty_semantic_results_are_not_bloom_rejections(self): + self.reconfigure(bloom=True) + result = self.store.retrieve(self.query) + self.assertEqual(result, []) + key = self.store.cache.key('detail', {'id': 1000}, 0) + self.assertFalse(self.client.exists(key)) + + def test_positive_and_negative_ttls_are_jittered(self): + for n in range(12): + self.store.cache.get("ttl", {"n": n}, lambda: {"value": True}) + ttls = [self.client.pttl(self.store.cache.key("ttl", {"n": n}, 0)) for n in range(12)] + self.assertTrue(all(47000 < ttl <= 72000 for ttl in ttls)) + self.assertGreater(max(ttls) - min(ttls), 500) + self.store.get(1) + self.assertTrue(15000 < self.client.pttl(self.store.cache.key("detail", {"id": 1}, 0)) <= 24000) + + def test_oversized_values_are_not_admitted(self): + self.reconfigure(maxBytes=1024) + calls = [] + for _ in range(2): + self.store.cache.get("large", {}, lambda: calls.append(1) or 'x' * 2048) + self.assertEqual(len(calls), 2) + + def test_request_rate_is_shared_between_store_instances(self): + self.reconfigure(requestsPerSecond=1) + # Freeze Redis's one-second bucket by starting near the beginning of a second. + while self.client.time()[1] > 100000: + time.sleep(0.01) + self.store.get(999) + with self.assertRaises(CacheBusy): + MemoryStore(self.root).get(998) + + def test_db_admission_blocks_other_process_and_releases_on_exit(self): + with self.store.db.cache_read_slot(1): + code = ("from pathlib import Path; import sys; from codex_agent.storage.database import WorkspaceDatabase; " + "from codex_agent.storage.cache import CacheBusy; " + "db=WorkspaceDatabase(Path(sys.argv[1]));\n" + "try:\n with db.cache_read_slot(1): print('unexpected')\n" + "except CacheBusy: print('blocked')\n") + result = subprocess.run([sys.executable, '-I', '-c', code, str(self.root)], + capture_output=True, text=True, timeout=10) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout.strip(), 'blocked') + with self.store.db.cache_read_slot(1): + pass + + def test_real_connection_failure_has_bounded_fallback(self): + os.environ['TRITON_REDIS_URL'] = 'unix://' + str(self.root / 'missing.sock') + store = MemoryStore(self.root) + self.assertEqual(store.stats()['total'], 0) + for _ in range(3): + store.stats() + with self.assertRaises(CacheBusy): + store.stats() + self.assertGreater(store.cache.backend.snapshot()['fallback'], 0) + + def test_oom_does_not_turn_successful_read_into_missing(self): + original = self.client.config_get('maxmemory')['maxmemory'] + self.client.config_set('maxmemory', 1) + try: + self.assertEqual(self.store.cache.get('oom', {}, lambda: {'real': True}), {'real': True}) + finally: + self.client.config_set('maxmemory', original) + self.backend.retry_at = 0 + + def test_credentials_are_not_given_to_generated_code(self): + self.assertNotIn('TRITON_REDIS_URL', validation_environment()) + self.assertNotIn('TRITON_REDIS_URL', self.document()) + + def test_bloom_rebuild_new_record_and_loss(self): + if not any(m.get('name', m.get(b'name')) in {'bf', b'bf'} for m in self.client.module_list()): + self.skipTest('Redis Bloom commands unavailable') + self.reconfigure(bloom=True) + self.store.add(record()) + self.assertEqual(self.store.warm_cache_ids()['ids'], 1) + with patch.object(self.store.db, 'rows', wraps=self.store.db.rows) as rows: + self.assertIsNone(self.store.get(99999)) + self.assertEqual(sum(c.args[0] is memories for c in rows.call_args_list), 0) + self.assertIsNotNone(self.store.get(1)) + new_id, _ = self.store.add(record(operator='new')) + self.assertIsNotNone(self.store.get(new_id)) + self.store.warm_cache_ids() + revision = self.store.db.revision('memory') + self.client.delete(f'{self.store.cache.prefix}:{revision}:ids') + self.assertIsNone(self.store.get(123456)) + + def test_malformed_entry_is_rebuilt(self): + key = self.store.cache.key('corrupt', {}, 0) + for value in ('not json', '{"wrong": 1}', '[]'): + self.client.set(key, value, ex=30) + self.assertEqual(self.store.cache.get('corrupt', {}, lambda: {'ok': True}), {'ok': True}) + + def test_eviction_policy_cannot_silently_evict_locks(self): + self.client.config_set('maxmemory-policy', 'allkeys-lfu') + self.backend.policy_until = 0 + try: + before = self.backend.snapshot().get('fallback', 0) + self.assertEqual(self.store.stats()['total'], 0) + self.assertEqual(self.backend.snapshot()['fallback'], before + 1) + finally: + self.client.config_set('maxmemory-policy', 'noeviction') + self.backend.retry_at = self.backend.policy_until = 0 + + def test_independent_processes_share_one_rebuild(self): + code = ("from pathlib import Path; import sys,time,secrets; " + "from codex_agent.memory import MemoryStore; root=Path(sys.argv[1]); s=MemoryStore(root);\n" + "def load():\n (root / ('rebuild-' + secrets.token_hex(8))).write_text('fixture'); " + "time.sleep(.2); return {'ok':True}\n" + "assert s.cache.get('process-hot', {}, load)=={'ok':True}\n") + def worker(_): + result = subprocess.run([sys.executable, '-I', '-c', code, str(self.root)], + capture_output=True, text=True, timeout=15) + self.assertEqual(result.returncode, 0, result.stderr) + with ThreadPoolExecutor(max_workers=6) as pool: + list(pool.map(worker, range(6))) + self.assertEqual(len(list(self.root.glob('rebuild-*'))), 1) + + def test_real_redis_stop_and_restart_recovers(self): + binary = os.environ.get('TRITON_TEST_REDIS_SERVER') + if not binary: + self.skipTest('temporary Redis executable not supplied') + sock = self.root / 'recovery.sock' + command = [binary, '--port', '0', '--unixsocket', str(sock), '--save', '', + '--appendonly', 'no', '--maxmemory', '64mb', '--maxmemory-policy', 'noeviction'] + processes = [] + def start(): + process = subprocess.Popen(command, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + processes.append(process) + client = redis.Redis(unix_socket_path=str(sock)) + for _ in range(100): + try: + client.ping() + return process + except redis.RedisError: + time.sleep(.02) + self.fail('Redis did not restart') + try: + process = start() + os.environ['TRITON_REDIS_URL'] = 'unix://' + str(sock) + store = MemoryStore(self.root) + self.assertEqual(store.stats()['total'], 0) + process.terminate() + process.wait(timeout=5) + self.assertEqual(store.stats()['total'], 0) + self.assertEqual(store.cache.backend.snapshot().get('fallback'), 1) + start() + time.sleep(2.1) + self.assertEqual(store.stats()['total'], 0) + self.assertEqual(store.stats()['total'], 0) + self.assertGreaterEqual(store.cache.backend.snapshot().get('hits', 0), 1) + finally: + for process in processes: + if process.poll() is None: + process.terminate() + process.wait(timeout=5) + + +class CacheConfigurationTests(unittest.TestCase): + def test_defaults_off_and_invalid_settings_rejected(self): + self.assertFalse(runtime_config({}).cache.enabled) + for cfg in ({'enabled': 'true'}, {'lockMs': 0}, {'maxConcurrent': 0}, + {'urlEnv': 'PATH'}, {'urlEnv': 'TRITON_MYSQL_URL'}, + {'bloomErrorRate': 0}, {'maxBytes': 0}, {'unknown': 1}): + with self.assertRaises(ValueError): + runtime_config({CONFIG_ENV: json.dumps({'schemaVersion': 1, + 'repoRoot': '/tmp/project', 'stateDir': '/tmp/state', 'cache': cfg})}) + + +if __name__ == '__main__': + unittest.main() diff --git a/packages/dsh-triton-riscv/python/codex_agent/tests/test_reference_library.py b/packages/dsh-triton-riscv/python/codex_agent/tests/test_reference_library.py index e5b854e..3198bb4 100644 --- a/packages/dsh-triton-riscv/python/codex_agent/tests/test_reference_library.py +++ b/packages/dsh-triton-riscv/python/codex_agent/tests/test_reference_library.py @@ -3,7 +3,8 @@ import json from pathlib import Path -import sqlite3 +from codex_agent.storage.database import WorkspaceDatabase +from codex_agent.storage.schema import references as catalog_table import tempfile import unittest @@ -260,20 +261,20 @@ def test_search_does_not_change_database_or_other_memory(self): production.write_bytes(b"original unrelated database") before = digest(production.read_bytes()) self.build() - catalog = self.output / "references.sqlite3" - original = digest(catalog.read_bytes()) + catalog = WorkspaceDatabase(self.output) + original = catalog.rows(catalog_table) search_library(self.output, "demo", environment=ENV) - self.assertEqual(original, digest(catalog.read_bytes())) + self.assertEqual(original, catalog.rows(catalog_table)) self.assertEqual(before, digest(production.read_bytes())) def test_search_rechecks_admission_metadata(self): self.build() - catalog = self.output / "references.sqlite3" - with sqlite3.connect(catalog) as connection: - row = connection.execute("SELECT id,payload FROM references_catalog").fetchone() - data = json.loads(row[1]) - data["decision"] = "quarantined" - connection.execute("UPDATE references_catalog SET payload=? WHERE id=?", (json.dumps(data), row[0])) + catalog = WorkspaceDatabase(self.output) + row = catalog.rows(catalog_table)[0] + data = json.loads(row["payload"]) + data["decision"] = "quarantined" + with catalog.transaction() as connection: + catalog.update(connection, catalog_table, {"payload": json.dumps(data)}, catalog_table.c.id == row["id"]) self.assertEqual(search_library(self.output, "demo", environment=ENV)["items"], []) def test_no_lexical_match_returns_no_reference(self): diff --git a/packages/dsh-triton-riscv/python/codex_agent/tests/test_runtime_config.py b/packages/dsh-triton-riscv/python/codex_agent/tests/test_runtime_config.py index c2b9b7a..f14a643 100644 --- a/packages/dsh-triton-riscv/python/codex_agent/tests/test_runtime_config.py +++ b/packages/dsh-triton-riscv/python/codex_agent/tests/test_runtime_config.py @@ -29,11 +29,11 @@ def test_native_defaults_are_read_only_and_preserve_rag(self): self.assertFalse(cfg.remote.requireTaskQuotas) def test_native_settings_are_authoritative_for_all_consumers(self): - from codex_agent.diagnostic_memory import memory_database_path + from codex_agent.diagnostic_memory import memory_workspace from codex_agent.paths import repository_root, state_root from codex_agent.remote_executor import RemoteValidationConfig raw = document(permissions={"validation": True}, - memory={"database": "/tmp/history.sqlite3"}) + storage={"urlEnv": "WORKSPACE_DB_URL"}) env = {CONFIG_ENV: raw, "TRITON_RISCV_REPO_ROOT": "/wrong", "TRITON_RISCV_STATE_DIR": "/wrong-state", "TRITON_RISCV_MEMORY_DB": "/wrong-db", "TRITON_RISCV_ALLOW_DEVELOPMENT_APPLY": "1", "TRITON_RISCV_ALLOW_REPAIR_APPLY": "1", @@ -42,7 +42,8 @@ def test_native_settings_are_authoritative_for_all_consumers(self): with patch.dict(os.environ, env, clear=True): self.assertEqual(repository_root(), Path("/tmp/operator repo").resolve()) self.assertEqual(state_root(Path("/unused")), Path("/tmp/operator-state").resolve()) - self.assertEqual(memory_database_path(Path("/unused")), Path("/tmp/history.sqlite3")) + self.assertEqual(memory_workspace(repository_root()), repository_root()) + self.assertEqual(runtime_config().storage.urlEnv, "WORKSPACE_DB_URL") self.assertIsNone(RemoteValidationConfig.from_env()) self.assertTrue(permission_enabled("validation")) self.assertTrue(permission_enabled("job")) @@ -113,7 +114,7 @@ def test_legacy_cli_configuration_remains_supported(self): cfg = runtime_config(env) self.assertEqual(cfg.repoRoot, "/tmp/old") self.assertTrue(cfg.permissions.repair) - self.assertEqual(cfg.memory.database, "history.sqlite3") + self.assertEqual(cfg.storage.urlEnv, "TRITON_MYSQL_URL") self.assertEqual(cfg.memory.embedding.tokenBudget, 256) self.assertEqual(cfg.remote.host, "old") with patch.dict(os.environ, env, clear=True): diff --git a/packages/dsh-triton-riscv/python/codex_agent/tests/test_task_queue.py b/packages/dsh-triton-riscv/python/codex_agent/tests/test_task_queue.py new file mode 100644 index 0000000..7b93e16 --- /dev/null +++ b/packages/dsh-triton-riscv/python/codex_agent/tests/test_task_queue.py @@ -0,0 +1,478 @@ +"""Use a disposable MySQL schema; RabbitMQ cases require TRITON_TEST_AMQP_URL.""" +import asyncio +from concurrent.futures import ThreadPoolExecutor +import json +import os +import subprocess +import sys +import time +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +from fastapi.testclient import TestClient + +from codex_agent.artifacts import ArtifactStore +from codex_agent.harness import HarnessAgent, HarnessSettings +from codex_agent.messaging.broker import Broker +from codex_agent.messaging.relay import publish_once +from codex_agent.platform.api import create_app +from codex_agent.platform.store import PlatformStore +from codex_agent.tasks.schema import jobs, outbox, attempts, inbox +from codex_agent.tasks.service import TaskService +from codex_agent.tasks.store import JobStore, Conflict, LeaseLost, now_ms +from codex_agent.tests.test_platform_service import FakeHarnessBackend +from codex_agent.workers.consumer import execute_delivery +from codex_agent.workers.handlers import execute_job + + +class Fixture: + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name).resolve() + self.cfg = {"schemaVersion": 1, "repoRoot": str(self.root), "stateDir": str(self.root / "agent-results"), "queue": {"enabled": True}, + "permissions": {"validation": True}} + env = patch.dict(os.environ, {"TRITON_RISCV_CONFIG": json.dumps(self.cfg), + "TRITON_AMQP_URL": os.environ.get("TRITON_TEST_AMQP_URL", "amqp://guest:guest@127.0.0.1:1/")}) + env.start() + self.addCleanup(env.stop) + self.store = JobStore(self.root) + + def submit(self, key="one", kind="validation"): + return self.store.submit(kind, {"fixture": True}, key) + + def event(self, job): + return self.store.db.rows(outbox, outbox.c.job_id == job["id"], order=(outbox.c.generation.desc(),))[0] + + def claim(self, job): + return self.store.claim(self.store.envelope(self.event(job)), job["kind"]) + + def expire(self, job): + with self.store.db.transaction() as conn: + self.store.db.update(conn, jobs, {"lease_until": 1}, jobs.c.id == job["id"]) + + +class TaskQueueTests(Fixture, unittest.TestCase): + def test_secrets_are_removed_from_generated_test_environment(self): + from codex_agent.process_control import validation_environment + with patch.dict(os.environ, {"TRITON_AMQP_URL": "amqp://private"}): + self.assertNotIn("TRITON_AMQP_URL", validation_environment()) + + def test_queue_capacity_rejects_without_partial_write(self): + self.store.config = self.store.config.model_copy(update={"maxPending": 1}) + self.submit() + with self.assertRaises(Conflict): + self.submit("second") + self.assertEqual(len(self.store.db.rows(jobs)), 1) + self.assertEqual(len(self.store.db.rows(outbox)), 1) + + def test_wrong_execution_host_defers_without_running(self): + job = self.submit() + with patch("codex_agent.tasks.store.socket.gethostname", return_value="different-machine"): + self.assertIsNone(self.claim(job)) + self.assertEqual(self.store.get(job["id"])["attempt"], 0) + self.assertEqual(self.store.get(job["id"])["generation"], 2) + self.assertEqual(self.store.get(job["id"])["result"]["reason"], "worker host mismatch") + + def test_validation_uses_existing_domain_guard_and_enqueues_memory(self): + from codex_agent.operator_lifecycle import validate_operator_target, decide_validation_plan + from codex_agent.validate_operator import OperatorValidationResult + from codex_agent.tests.test_operator_lifecycle import ORIGINAL_SOURCE, TEST_SOURCE + folder = self.root / "python/examples/flaggems" + folder.mkdir(parents=True) + (folder / "demo.py").write_text(ORIGINAL_SOURCE) + (folder / "test_demo.py").write_text(TEST_SOURCE) + plan = validate_operator_target(self.root, "demo", source_env=False) + decide_validation_plan(self.root, plan.run_id, approve=True, reviewer="fixture") + job = TaskService(self.root).submit_validation("demo", plan.run_id, "validate") + claim = self.claim(job) + log = self.root / "synthetic-validation.log" + log.write_text("Synthetic hardware boundary fixture; not a RISC-V execution.\n1 passed\n") + fake = OperatorValidationResult(operator="demo", implementation_file="python/examples/flaggems/demo.py", + test_files=["python/examples/flaggems/test_demo.py"], command=plan.command, dry_run=False, + exit_code=0, status="passed", failure_stage=None, likely_reason=None, error_excerpt=[], + duration_seconds=.1, log_path=str(log)) + with patch("codex_agent.operator_lifecycle.run_operator", return_value=fake), \ + patch("codex_agent.operator_lifecycle.remember_validation") as inline_memory: + execute_job(self.root, job["id"], claim["owner"]) + inline_memory.assert_not_called() + finished = self.store.get(job["id"]) + self.assertEqual(finished["status"], "succeeded", finished) + self.assertEqual(finished["result"]["memory_write"]["status"], "deferred") + followup = self.store.db.rows(jobs, jobs.c.kind == "memory") + self.assertEqual(len(followup), 1) + self.assertEqual(len(self.store.detail(job["id"])["artifacts"]), 3) + memory = self.claim(self.store.public(followup[0])) + execute_job(self.root, memory["id"], memory["owner"]) + self.assertEqual(self.store.get(memory["id"])["status"], "succeeded") + self.assertEqual(self.store.detail(job["id"])["followups"][0]["status"], "succeeded") + + def test_task_and_outbox_rollback_together(self): + with self.assertRaisesRegex(RuntimeError, "rollback"): + with self.store.db.transaction() as conn: + self.store.submit("agent", {}, "rollback", connection=conn) + raise RuntimeError("rollback") + self.assertEqual(self.store.db.rows(jobs), []) + self.assertEqual(self.store.db.rows(outbox), []) + + def test_concurrent_idempotency_and_conflicting_input(self): + with ThreadPoolExecutor(6) as pool: + results = list(pool.map(lambda _: self.submit(), range(20))) + self.assertEqual(len({r["id"] for r in results}), 1) + self.assertEqual(len(self.store.db.rows(outbox)), 1) + with self.assertRaises(Conflict): + self.store.submit("validation", {"different": True}, "one") + + def test_invalid_kind_payload_and_key(self): + for kind, payload, key in [("shell", {}, "k"), ("agent", {}, ""), ("agent", {"x": "x" * 40000}, "k")]: + with self.assertRaises(ValueError): + self.store.submit(kind, payload, key) + + def test_duplicate_delivery_claims_one_attempt(self): + job = self.submit() + with ThreadPoolExecutor(6) as pool: + claims = list(pool.map(lambda _: self.claim(job), range(12))) + self.assertEqual(sum(c is not None for c in claims), 1) + self.assertEqual(len(self.store.db.rows(attempts)), 1) + claim = next(c for c in claims if c) + self.store.finish(job["id"], claim["owner"], "succeeded", {"fixture": "done"}) + self.assertIsNone(self.claim(job)) + + def test_envelope_is_reference_not_authority(self): + job = self.submit() + envelope = self.store.envelope(self.event(job)) + self.assertNotIn("payload", envelope) + for bad in [{**envelope, "job_id": "wrong"}, {**envelope, "command": "touch unsafe"}, + {**envelope, "workspace_id": "other"}, {**envelope, "version": True}]: + with self.assertRaises((ValueError, KeyError)): + self.store.claim(bad, "validation") + self.assertEqual(self.store.get(job["id"])["attempt"], 0) + + def test_other_workspace_cannot_read_task(self): + job = self.submit() + with self.assertRaises(KeyError): + JobStore(self.root / "other").get(job["id"]) + + def test_mutations_are_fifo_and_memory_is_independent(self): + first, second = self.submit("first"), self.submit("second") + self.assertIsNone(self.claim(second)) + running = self.claim(first) + self.assertIsNone(self.claim(second)) + memory = self.submit("memory", "memory") + self.assertIsNotNone(self.claim(memory)) + self.store.finish(first["id"], running["owner"], "succeeded", {}) + self.assertIsNotNone(self.claim(second)) + + def test_no_effects_recovery_requeues_but_old_owner_cannot_write(self): + job = self.submit() + old = self.claim(job) + self.expire(job) + self.assertEqual(self.store.recover()[0]["status"], "queued") + fresh = self.claim(job) + self.assertNotEqual(old["owner"], fresh["owner"]) + with self.assertRaises(LeaseLost): + self.store.finish(job["id"], old["owner"], "succeeded", {}) + + def test_effectful_crash_blocks_replay_and_requires_explicit_reconciliation(self): + job = self.submit() + claim = self.claim(job) + self.store.begin_effects(job["id"], claim["owner"]) + self.expire(job) + self.assertEqual(self.store.recover()[0]["status"], "needs-reconciliation") + self.assertIsNone(self.claim(job)) + with self.assertRaises(ValueError): + self.store.reconcile(job["id"], "") + result = self.store.reconcile(job["id"], "fixture external process checked and stopped") + self.assertEqual(result["status"], "failed") + self.assertIn("operator-attested", result["result"]["verification"]) + + def test_retry_budget_eventually_dead(self): + job = self.submit() + for _ in range(3): + self.claim(job) + self.expire(job) + self.store.recover() + self.assertEqual(self.store.get(job["id"])["status"], "dead") + self.assertEqual(len(self.store.db.rows(attempts)), 3) + + def test_cancel_queued_never_executes_and_running_is_only_requested(self): + job = self.submit() + self.assertEqual(self.store.cancel(job["id"])["status"], "cancelled") + self.assertIsNone(self.claim(job)) + other = self.submit("other") + claim = self.claim(other) + self.assertEqual(self.store.cancel(other["id"])["status"], "running") + with self.assertRaises(LeaseLost): + self.store.begin_effects(other["id"], claim["owner"]) + + def test_outbox_concurrent_publishers_and_expired_publisher(self): + self.submit() + with ThreadPoolExecutor(4) as pool: + claims = list(pool.map(lambda _: self.store.claim_publication(), range(8))) + self.assertEqual(sum(c is not None for c in claims), 1) + event = next(c for c in claims if c) + with self.store.db.transaction() as conn: + self.store.db.update(conn, outbox, {"lease_until": 1}, outbox.c.id == event["id"]) + newer = self.store.claim_publication() + with self.assertRaises(LeaseLost): + self.store.publication_result(event) + self.store.publication_result(newer) + self.assertEqual(self.store.db.rows(outbox)[0]["status"], "published") + + def test_artifacts_bound_to_attempt_and_checksum(self): + job = self.submit() + claim = self.claim(job) + storage = ArtifactStore(self.root) + identifier = storage.put(self.store, job["id"], claim["owner"], "result.json", b'{"ok":true}') + record = self.store.detail(job["id"])["artifacts"][0] + self.assertEqual(record["id"], identifier) + self.assertEqual(storage.read(record), b'{"ok":true}') + with self.assertRaises(ValueError): + storage.put(self.store, job["id"], claim["owner"], "../bad", b"no") + (storage.folder / record["storage_key"]).write_bytes(b"corrupted") + with self.assertRaises(ValueError): + storage.read(record) + + def test_finish_and_memory_followup_atomic(self): + job = self.submit() + claim = self.claim(job) + with patch.object(self.store, "submit", side_effect=RuntimeError("db unavailable")): + with self.assertRaises(RuntimeError): + self.store.finish(job["id"], claim["owner"], "succeeded", {}, followup={"receipt": "fixture"}) + self.assertEqual(self.store.get(job["id"])["status"], "running") + self.store.finish(job["id"], claim["owner"], "succeeded", {}, followup={"receipt": "fixture"}) + self.assertEqual(len(self.store.db.rows(jobs)), 2) + self.assertEqual(len(self.store.db.rows(outbox)), 2) + + def test_validation_needs_real_approval_and_rechecks_changed_inputs(self): + from codex_agent.operator_lifecycle import validate_operator_target, decide_validation_plan + from codex_agent.tests.test_operator_lifecycle import ORIGINAL_SOURCE, TEST_SOURCE + folder = self.root / "python/examples/flaggems" + folder.mkdir(parents=True) + implementation = folder / "demo.py" + implementation.write_text(ORIGINAL_SOURCE) + (folder / "test_demo.py").write_text(TEST_SOURCE) + plan = validate_operator_target(self.root, "demo", source_env=False) + service = TaskService(self.root) + with self.assertRaises(PermissionError): + service.submit_validation("demo", plan.run_id, "validate") + decide_validation_plan(self.root, plan.run_id, approve=True, reviewer="fixture") + job = service.submit_validation("demo", plan.run_id, "validate") + claim = self.claim(job) + implementation.write_text(ORIGINAL_SOURCE + "\n# changed after approval\n") + with patch("codex_agent.operator_lifecycle.validate_operator_target") as execute: + execute_job(self.root, job["id"], claim["owner"]) + execute.assert_not_called() + self.assertEqual(self.store.get(job["id"])["status"], "failed") + + def test_http_submission_is_durable_and_duplicate_does_not_add_message(self): + agent = HarnessAgent(HarnessSettings.from_env(self.root, {}), FakeHarnessBackend()) + app = create_app(self.root, harness_agent=agent) + with TestClient(app) as client: + session = client.post("/api/sessions", json={"title": "fixture"}).json() + url = f'/api/sessions/{session["id"]}/messages' + payload = {"content": "inspect demo", "request_id": "request-1"} + response = client.post(url, json=payload) + self.assertEqual(response.status_code, 202, response.text) + first = response.json() + self.assertEqual(first["run"]["status"], "queued") + self.assertEqual(client.post(url, json=payload).json()["job"]["id"], first["job"]["id"]) + self.assertEqual(len(app.state.store.list_messages(session["id"])), 1) + self.assertEqual(client.post(url, json={**payload, "content": "different"}).status_code, 409) + self.assertEqual(client.post(url, json={"content": "no key"}).status_code, 400) + job = first["job"] + claim = self.claim(job) + with patch("codex_agent.harness.HarnessAgent", return_value=agent): + execute_job(self.root, job["id"], claim["owner"]) + detail = client.get('/api/jobs/' + job["id"]).json() + self.assertEqual(detail["status"], "succeeded", detail) + run = client.get('/api/runs/' + first["run"]["id"]).json() + self.assertEqual(run["status"], "completed") + self.assertGreater(len(client.get('/api/runs/' + run["id"] + '/events').json()), 1) + + def test_completed_run_recovers_without_second_model_call(self): + platform = PlatformStore(self.root) + session = platform.create_session() + from types import SimpleNamespace + submitted = TaskService(self.root).submit_turn(SimpleNamespace(store=platform), session["id"], "fixture", "once") + job = submitted["job"] + claim = self.claim(job) + self.store.begin_effects(job["id"], claim["owner"]) + platform.update_run(submitted["run"]["id"], status="completed", result={"fixture": "committed"}) + self.expire(job) + self.assertEqual(self.store.recover()[0]["status"], "succeeded") + self.assertEqual(self.store.get(job["id"])["result"], {"fixture": "committed"}) + + def test_queued_cancellation_emits_terminal_event_for_sse(self): + from types import SimpleNamespace + platform = PlatformStore(self.root) + session = platform.create_session() + submitted = TaskService(self.root).submit_turn(SimpleNamespace(store=platform), session["id"], "fixture", "once") + self.store.cancel(submitted["job"]["id"]) + events = platform.list_events(submitted["run"]["id"]) + self.assertEqual(events[-1]["event_type"], "cancelled") + self.assertEqual(events[-1]["payload"]["job_id"], submitted["job"]["id"]) + + +@unittest.skipUnless(os.environ.get("TRITON_TEST_AMQP_URL"), "temporary RabbitMQ required") +class RabbitMQTests(Fixture, unittest.IsolatedAsyncioTestCase): + async def test_separate_relay_and_worker_cli_processes(self): + job = self.store.submit("memory", {"receipt_path": "missing.json", "sha256": "0" * 64}, "cli") + for role in ("relay", "memory"): + result = await asyncio.to_thread(subprocess.run, + [sys.executable, "-I", "-m", "codex_agent.workers", role, "--workspace", str(self.root), "--once"], + capture_output=True, timeout=30) + self.assertEqual(result.returncode, 0, result.stderr.decode()) + detail = self.store.detail(job["id"]) + self.assertEqual(detail["deliveries"][0]["status"], "published") + self.assertEqual(detail["status"], "failed") + self.assertEqual(detail["result"]["error_type"], "ValueError") + + async def test_timeout_stops_real_child_and_does_not_claim_success(self): + job = self.submit() + original = asyncio.create_subprocess_exec + children = [] + async def controlled_child(*_args, **kwargs): + child = await original(sys.executable, "-c", "import time; time.sleep(60)", **kwargs) + children.append(child) + return child + # Only shorten this fixture's wall deadline, not production configuration. + self.store.config = self.store.config.model_copy(update={"jobTimeoutSeconds": .1}) + async with Broker(self.store) as broker: + await publish_once(self.store, broker) + delivery = await broker.queues["validation"].get(timeout=5) + with patch("codex_agent.workers.consumer.asyncio.create_subprocess_exec", side_effect=controlled_child): + await execute_delivery(self.store, delivery, "validation") + self.assertTrue(delivery.processed) + self.assertIsNotNone(children[0].returncode) + self.assertEqual(self.store.get(job["id"])["status"], "failed") + + @unittest.skipUnless(os.environ.get("TRITON_TEST_RABBITMQ_CTL"), "private broker control required") + async def test_broker_restart_preserves_confirmed_message(self): + job = self.submit() + async with Broker(self.store) as broker: + await publish_once(self.store, broker) + env = dict(os.environ) + env["PATH"] = str(Path(env["ERL_ROOTDIR"]) / "bin") + ":" + env["PATH"] + ctl = env["TRITON_TEST_RABBITMQ_CTL"] + stopped = await asyncio.to_thread(subprocess.run, [ctl, "stop_app"], env=env, capture_output=True, timeout=30) + self.assertEqual(stopped.returncode, 0, stopped.stderr.decode()) + try: + with self.assertRaises(RuntimeError): + async with Broker(self.store): + self.fail("stopped broker must not connect") + finally: + started = await asyncio.to_thread(subprocess.run, [ctl, "start_app"], env=env, capture_output=True, timeout=30) + self.assertEqual(started.returncode, 0, started.stderr.decode()) + async with Broker(self.store) as broker: + delivery = await broker.queues["validation"].get(timeout=5) + self.assertEqual(json.loads(delivery.body)["job_id"], job["id"]) + await delivery.ack() + + async def test_confirm_before_database_checkpoint_can_republish_safely(self): + job = self.submit() + async with Broker(self.store) as broker: + event = self.store.claim_publication() + await broker.publish(event) + with self.store.db.transaction() as conn: + self.store.db.update(conn, outbox, {"lease_until": 1}, outbox.c.id == event["id"]) + await publish_once(self.store, broker) + deliveries = [await broker.queues["validation"].get(timeout=5) for _ in range(2)] + claims = [self.store.claim(json.loads(item.body), "validation") for item in deliveries] + self.assertEqual(sum(c is not None for c in claims), 1) + for delivery in deliveries: + await delivery.ack() + self.assertEqual(self.event(job)["attempts"], 2) + + async def test_backlog_delivers_all_100_distinct_jobs(self): + started = time.monotonic() + for number in range(100): + self.submit(str(number), "memory") + async with Broker(self.store) as broker: + for _ in range(100): + await publish_once(self.store, broker) + seen = set() + for _ in range(100): + message = await broker.queues["memory"].get(timeout=5) + envelope = json.loads(message.body) + claim = self.store.claim(envelope, "memory") + self.store.finish(claim["id"], claim["owner"], "succeeded", {"synthetic_queue_fixture": True}) + seen.add(claim["id"]) + await message.ack() + self.assertEqual(len(seen), 100) + print(json.dumps({"fixture": "queue-only-no-model-no-operator", "submitted": 100, "completed": len(seen), + "seconds": round(time.monotonic() - started, 3)}), flush=True) + + async def test_confirmed_publish_delivery_and_duplicate(self): + job = self.submit() + async with Broker(self.store) as broker: + await publish_once(self.store, broker) + self.assertEqual(self.event(job)["status"], "published") + delivery = await broker.queues["validation"].get(timeout=5) + claim = self.store.claim(json.loads(delivery.body), "validation") + self.store.finish(job["id"], claim["owner"], "succeeded", {"fixture": True}) + await delivery.ack() + await broker.publish(self.event(job)) + duplicate = await broker.queues["validation"].get(timeout=5) + await execute_delivery(self.store, duplicate, "validation") + self.assertEqual(len(self.store.db.rows(attempts)), 1) + + async def test_unroutable_message_is_not_marked_published(self): + job = self.submit() + async with Broker(self.store) as broker: + await broker.queues["validation"].unbind(broker.exchange, routing_key="validation") + await publish_once(self.store, broker) + event = self.event(job) + self.assertEqual(event["status"], "pending") + self.assertTrue(event["last_error"]) + self.assertIsNone(event["published_ms"]) + + async def test_connection_failure_keeps_outbox_for_retry(self): + self.submit() + async with Broker(self.store) as broker: + await broker.connection.close() + await publish_once(self.store, broker) + self.assertEqual(self.store.db.rows(outbox)[0]["status"], "pending") + + async def test_poison_message_goes_to_dead_letter_without_execution(self): + import aio_pika + async with Broker(self.store) as broker: + await broker.exchange.publish(aio_pika.Message(b'{"command":"not allowed"}'), routing_key="agent") + delivery = await broker.queues["agent"].get(timeout=5) + await execute_delivery(self.store, delivery, "agent") + dead = await broker.channel.get_queue(broker.prefix + ".dead") + message = await dead.get(timeout=5) + self.assertIn(b"not allowed", message.body) + await message.ack() + self.assertEqual(self.store.db.rows(attempts), []) + + async def test_child_process_records_invalid_receipt_failure_before_ack(self): + job = self.store.submit("memory", {"receipt_path": "missing.json", "sha256": "0" * 64}, "invalid") + async with Broker(self.store) as broker: + await publish_once(self.store, broker) + delivery = await broker.queues["memory"].get(timeout=5) + await execute_delivery(self.store, delivery, "memory") + self.assertTrue(delivery.processed) + detail = self.store.detail(job["id"]) + self.assertEqual(detail["status"], "failed", detail) + self.assertEqual(len(detail["attempts"]), 1) + self.assertEqual(self.store.db.rows(inbox)[0]["status"], "completed") + + async def test_unacked_delivery_redelivered_after_channel_close(self): + job = self.submit() + async with Broker(self.store) as broker: + await publish_once(self.store, broker) + first = await broker.queues["validation"].get(timeout=5) + first_id = first.message_id + async with Broker(self.store) as broker: + again = await broker.queues["validation"].get(timeout=5) + self.assertEqual(again.message_id, first_id) + self.assertTrue(again.redelivered) + await again.ack() + self.assertEqual(self.store.get(job["id"])["status"], "queued") + + +if __name__ == "__main__": + unittest.main() diff --git a/packages/dsh-triton-riscv/python/codex_agent/workers/__init__.py b/packages/dsh-triton-riscv/python/codex_agent/workers/__init__.py new file mode 100644 index 0000000..54d7c05 --- /dev/null +++ b/packages/dsh-triton-riscv/python/codex_agent/workers/__init__.py @@ -0,0 +1 @@ +"""Independent worker processes for agent, validation and evidence-memory jobs.""" diff --git a/packages/dsh-triton-riscv/python/codex_agent/workers/__main__.py b/packages/dsh-triton-riscv/python/codex_agent/workers/__main__.py new file mode 100644 index 0000000..d7d02d7 --- /dev/null +++ b/packages/dsh-triton-riscv/python/codex_agent/workers/__main__.py @@ -0,0 +1,85 @@ +"""Run independent RabbitMQ relay, recovery and typed workers.""" +import argparse +import asyncio +import json +import signal +from pathlib import Path + +from codex_agent.runtime_config import runtime_config +from codex_agent.tasks.store import JobStore +from codex_agent.messaging.broker import Broker +from codex_agent.messaging.relay import publish_once +from .consumer import execute_delivery + + +async def serve(args): + root = args.workspace.resolve() + config = runtime_config() + if not config.queue.enabled or (config.repoRoot and Path(config.repoRoot).resolve() != root): + raise ValueError("enable queue and select the configured workspace") + jobs = JobStore(root) + if args.role == 'reconcile': + if not args.confirm_no_active_execution or not args.job or not args.note: + raise ValueError('reconcile requires --job, --note and --confirm-no-active-execution') + print(json.dumps(jobs.reconcile(args.job, args.note))) + return + if args.role == 'execute': + from .handlers import execute_job + execute_job(root, args.job, args.owner) + return + if args.role == 'recover': + while True: + result = await asyncio.to_thread(jobs.recover) + if result: + print(json.dumps(result), flush=True) + if args.once: + return + await asyncio.sleep(5) + while True: + try: + async with Broker(jobs) as broker: + if args.role == 'relay': + while True: + sent = await publish_once(jobs, broker) + if args.once: + return + if not sent: + await asyncio.sleep(.5) + else: + async with broker.queues[args.role].iterator() as deliveries: + async for delivery in deliveries: + await execute_delivery(jobs, delivery, args.role) + if args.once: + return + except Exception as error: + print(json.dumps({'status': 'unavailable', 'component': args.role, + 'error_type': type(error).__name__}), flush=True) + if args.once: + raise SystemExit(1) from None + await asyncio.sleep(3) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('role', choices=['relay', 'recover', 'agent', 'validation', 'memory', 'execute', 'reconcile']) + parser.add_argument('--workspace', type=Path, required=True) + parser.add_argument('--once', action='store_true') + parser.add_argument('--job') + parser.add_argument('--owner') + parser.add_argument('--note') + parser.add_argument('--confirm-no-active-execution', action='store_true') + args = parser.parse_args() + if args.role == 'execute' and (not args.job or not args.owner): + parser.error('execute requires job and owner') + try: + async def controlled(): + task = asyncio.current_task() + asyncio.get_running_loop().add_signal_handler(signal.SIGTERM, task.cancel) + await serve(args) + asyncio.run(controlled()) + except (KeyboardInterrupt, asyncio.CancelledError): + pass + + +if __name__ == '__main__': + main() diff --git a/packages/dsh-triton-riscv/python/codex_agent/workers/consumer.py b/packages/dsh-triton-riscv/python/codex_agent/workers/consumer.py new file mode 100644 index 0000000..09d3941 --- /dev/null +++ b/packages/dsh-triton-riscv/python/codex_agent/workers/consumer.py @@ -0,0 +1,74 @@ +"""Bounded child execution, manual ACK and MySQL-owned recovery.""" +import asyncio +import json +import os +import signal +import sys +import time + +from codex_agent.tasks.store import LeaseLost + + +async def execute_delivery(jobs, delivery, kind): + try: + if len(delivery.body) > 4096: + raise ValueError("oversized task envelope") + envelope = json.loads(delivery.body) + job = await asyncio.to_thread(jobs.claim, envelope, kind) + except (ValueError, KeyError): + await delivery.reject(requeue=False) + return + if job is None: + await delivery.ack() + return + process = None + try: + # No credentials, code or shell command arrive in the queue message. + process = await asyncio.create_subprocess_exec(sys.executable, '-I', '-m', + 'codex_agent.workers', 'execute', '--workspace', str(jobs.db.root), + '--job', job['id'], '--owner', job['owner'], start_new_session=True, + stdout=asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.DEVNULL) + deadline = time.monotonic() + jobs.config.jobTimeoutSeconds + while process.returncode is None: + cancelled = await asyncio.to_thread(jobs.heartbeat, job['id'], job['owner']) + if cancelled or time.monotonic() > deadline: + break + try: + await asyncio.wait_for(process.wait(), timeout=min(5, jobs.config.leaseSeconds / 3, + max(.01, deadline - time.monotonic()))) + except asyncio.TimeoutError: + pass + if process.returncode is None: + await stop_child(process) + current = await asyncio.to_thread(jobs.get, job['id']) + if current['status'] == 'running': + status = 'needs-reconciliation' if current['effects_started'] else ( + 'cancelled' if current['cancel_requested'] else 'failed') + await asyncio.to_thread(jobs.finish, job['id'], job['owner'], status, + {'reason': 'child stopped without a durable result', 'exit_code': process.returncode}) + await delivery.ack() + except LeaseLost: + if process is not None: + await stop_child(process) + # Durable recovery owns the task now; do not replay it just because AMQP redelivers. + await delivery.ack() + except BaseException: + if process is not None: + await stop_child(process) + raise # Connection closes without ACK; recovery handles the persisted claim. + + +async def stop_child(process): + try: + os.killpg(process.pid, signal.SIGTERM) + except ProcessLookupError: + pass + try: + await asyncio.wait_for(process.wait(), 3) + except asyncio.TimeoutError: + pass + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + await process.wait() diff --git a/packages/dsh-triton-riscv/python/codex_agent/workers/handlers.py b/packages/dsh-triton-riscv/python/codex_agent/workers/handlers.py new file mode 100644 index 0000000..1b80fea --- /dev/null +++ b/packages/dsh-triton-riscv/python/codex_agent/workers/handlers.py @@ -0,0 +1,100 @@ +"""Reuse domain services; do not grant capabilities just because work arrived via AMQP.""" +from dataclasses import replace +import hashlib +import json + +from codex_agent.artifacts import ArtifactStore +from codex_agent.tasks.store import JobStore, LeaseLost, encode + + +def execute_job(root, identifier, owner): + jobs = JobStore(root) + with jobs.db.transaction() as conn: + job = jobs.require_owner(conn, identifier, owner) + payload = json.loads(job["payload_json"]) + artifacts = ArtifactStore(root) + followup = None + try: + if job["kind"] == "agent": + from codex_agent.harness import HarnessAgent, HarnessSettings + from codex_agent.platform.executor import HarnessRunExecutor + from codex_agent.platform.store import PlatformStore + store = PlatformStore(root) + store.db.task_guard = lambda conn: jobs.require_owner(conn, identifier, owner) + settings = replace(HarnessSettings.from_env(root), managed_context=True) + agent = HarnessAgent(settings) + executor = HarnessRunExecutor(root, store, agent, workers=0) + jobs.begin_effects(identifier, owner) + try: + executor._execute(payload["run_id"]) + finally: + executor.close() + run = store.get_run(payload["run_id"]) + result = run["result"] + status = "succeeded" if run["status"] == "completed" else "failed" + elif job["kind"] == "validation": + from codex_agent.execution_guard import artifact, approval_digest + from codex_agent.operator_lifecycle import validate_operator_target + from codex_agent.runtime_config import remote_execution_identity + from codex_agent.validation_evidence import capture_source_snapshot + plan = artifact(root, "validation", payload["approved_run_id"]) + if (approval_digest(root, "validation", plan) != payload["approval_digest"] + or remote_execution_identity() != payload["remote_identity"] + or plan.get("approval", {}).get("status") != "approved" + or capture_source_snapshot(root, plan["implementation_file"], plan["test_files"]) != plan.get("source_snapshot")): + raise PermissionError("queued validation input or target changed; request new approval") + jobs.begin_effects(identifier, owner) + from codex_agent.tasks.context import defer_memory + from codex_agent.validation_evidence import audit_validation_receipt + token = defer_memory.set(True) + try: + value = validate_operator_target(root, payload["operator"], execute=True, + approved_run_id=payload["approved_run_id"], source_env=payload["source_env"], + timeout_seconds=payload["timeout_seconds"]) + finally: + defer_memory.reset(token) + result = value.model_dump() + status = "succeeded" if result["status"] == "passed" else "failed" + receipt = (root / result["receipt_path"]).resolve() + receipt.relative_to(root) + audited = audit_validation_receipt(root, result) + if audited.verdict in {"verified-passed", "verified-failed"}: + followup = {"receipt_path": receipt.relative_to(root).as_posix(), + "sha256": hashlib.sha256(receipt.read_bytes()).hexdigest()} + artifacts.put(jobs, identifier, owner, "receipt.json", receipt.read_bytes()) + if result.get("log_path"): + log = (root / result["log_path"]).resolve() + log.relative_to(root) + if log.is_file() and log.stat().st_size <= 16 * 1024 * 1024: + artifacts.put(jobs, identifier, owner, "validation.log", log.read_bytes()) + else: + from codex_agent.diagnostic_memory import remember_validation + from codex_agent.validation_evidence import audit_validation_receipt + path = (root / payload["receipt_path"]).resolve() + path.relative_to(root / "agent-results/operator-lifecycle/receipts") + if path.stat().st_size > 4 * 1024 * 1024: + raise ValueError("receipt too large") + raw = path.read_bytes() + if hashlib.sha256(raw).hexdigest() != payload["sha256"]: + raise ValueError("receipt changed since submission") + receipt = json.loads(raw) + audited = audit_validation_receipt(root, receipt) + if audited.verdict not in {"verified-passed", "verified-failed"}: + raise ValueError("receipt is not verified execution evidence") + jobs.begin_effects(identifier, owner) + result = remember_validation(root, receipt) + if result.get("status") == "unavailable": + result = {"status": "unavailable", "reason": "memory storage unavailable; original validation unchanged"} + status = "succeeded" if result["status"] in {"recorded", "deduplicated"} else "failed" + artifacts.put(jobs, identifier, owner, "result.json", encode(result).encode()) + jobs.finish(identifier, owner, status, result, followup=followup) + except LeaseLost: + raise + except Exception as error: + # Never return an arbitrary SDK exception which could contain a credential URL. + result = {"error_type": type(error).__name__, "reason": "Worker failed; inspect task artifacts and evidence"} + if isinstance(error, PermissionError): + result["reason"] = "Validation inputs or authorization changed; review a new plan. Unknown execution still requires reconciliation." + current = jobs.get(identifier) + status = "needs-reconciliation" if current["effects_started"] and job["kind"] != "memory" else "failed" + jobs.finish(identifier, owner, status, result) diff --git a/packages/dsh-triton-riscv/python/pyproject.toml b/packages/dsh-triton-riscv/python/pyproject.toml index 8c9996c..9f0269c 100644 --- a/packages/dsh-triton-riscv/python/pyproject.toml +++ b/packages/dsh-triton-riscv/python/pyproject.toml @@ -8,7 +8,7 @@ version = "0.1.0" description = "Triton-RISCV domain tools, evidence memory and workbench for DeepSeek Harness" requires-python = ">=3.10" license = {file = "LICENSE"} -dependencies = ["pydantic>=2.11,<3", "mcp==2.1.1", "fastapi>=0.115", "httpx>=0.28", "uvicorn>=0.34", "pytest>=8,<10"] +dependencies = ["pydantic>=2.11,<3", "mcp==2.1.1", "fastapi>=0.115", "httpx>=0.28", "uvicorn>=0.34", "pytest>=8,<10", "SQLAlchemy>=2.0,<2.1", "PyMySQL[rsa]>=1.1,<2", "alembic>=1.13,<2", "redis>=5,<7", "aio-pika>=9.5,<10"] [project.optional-dependencies] workbench = ["deepseek-harness-sdk==0.1.1rc1", "openai>=2,<3"] @@ -17,6 +17,7 @@ embedding = ["sentence-transformers>=3", "tokenizers>=0.20", "openai>=2,<3"] [project.scripts] triton-riscv-workbench = "codex_agent.platform.__main__:main" triton-riscv-mcp = "codex_agent.harness.mcp_server:main" +triton-riscv-worker = "codex_agent.workers.__main__:main" [tool.setuptools.packages.find] where = ["."] @@ -28,4 +29,4 @@ testpaths = ["codex_agent/tests"] norecursedirs = ["fixtures"] [tool.setuptools.package-data] -codex_agent = ["*.json", "README.md", "specs/*.json", "tests/fixtures/*.json", "harness/*.yml", "harness/bundle/*", "harness/bundle/lib/*.js", "harness/bundle/prompts/*.md", "frontend/dist/*", "frontend/dist/assets/*"] +codex_agent = ["*.json", "README.md", "specs/*.json", "tests/fixtures/*.json", "tests/fixtures/*.sql", "harness/*.yml", "harness/bundle/*", "harness/bundle/lib/*.js", "harness/bundle/prompts/*.md", "frontend/dist/*", "frontend/dist/assets/*", "storage/migrations/*.py", "storage/migrations/versions/*.py"] diff --git a/packages/dsh-triton-riscv/scripts/local-environment.mjs b/packages/dsh-triton-riscv/scripts/local-environment.mjs index 96b8cdf..3e1957c 100644 --- a/packages/dsh-triton-riscv/scripts/local-environment.mjs +++ b/packages/dsh-triton-riscv/scripts/local-environment.mjs @@ -4,7 +4,7 @@ import { parseEnv } from 'node:util' const localKeys = new Set([ 'DSH_NATIVE_SOURCE', 'DSH_HOME', 'DSH_MODEL', 'DSH_PROVIDER', 'ISRC_BASE_URL', - 'TRITON_RISCV_REPO_ROOT', 'TRITON_RISCV_STATE_DIR', 'TRITON_RISCV_MEMORY_DB', + 'TRITON_RISCV_REPO_ROOT', 'TRITON_RISCV_STATE_DIR', 'TRITON_RISCV_MCP_PYTHON', 'TRITON_RISCV_NATIVE_PORT', 'TRITON_RISCV_KEYCHAIN_SERVICE', 'TRITON_RISCV_BROWSER_DIRECTORY_PICKER', 'TRITON_RISCV_MEMORY_RETRIEVAL_MODE', 'TRITON_RISCV_MEMORY_CONTEXT_FORMAT', 'TRITON_RISCV_EMBEDDING_PROVIDER', @@ -20,6 +20,8 @@ export function loadLocalEnvironment(root, ambient = process.env) { if (existsSync(settingsPath)) { const local = JSON.parse(readFileSync(settingsPath, 'utf8')) for (const [key, value] of Object.entries(local)) { + if (key === 'TRITON_RISCV_MEMORY_DB') + throw new Error('Legacy memory database setting: import SQLite into MySQL, then remove TRITON_RISCV_MEMORY_DB from .state/native-local.json') if (!localKeys.has(key) || typeof value !== 'string') throw new Error('Invalid local configuration key: ' + key) if (env[key] === undefined) env[key] = value diff --git a/packages/dsh-triton-riscv/tests/integration/native-host.spec.ts b/packages/dsh-triton-riscv/tests/integration/native-host.spec.ts index 3d5e0a6..1b0173b 100644 --- a/packages/dsh-triton-riscv/tests/integration/native-host.spec.ts +++ b/packages/dsh-triton-riscv/tests/integration/native-host.spec.ts @@ -2,6 +2,18 @@ import { expect, it } from 'vitest' import { spawnSync } from 'node:child_process' import { dirname, resolve } from 'node:path' import { fileURLToPath } from 'node:url' +import { stripVTControlCharacters } from 'node:util' + +const passedTests = /\bTests\s+[1-9]\d* passed\b/ + +it.each([ + ['plain', 'Tests 12 passed (12)', true], + ['colored', '\u001b[2m Tests \u001b[22m \u001b[1m\u001b[32m12 passed\u001b[39m (12)', true], + ['empty', 'Tests 0 passed (0)', false], + ['files only', 'Test Files 1 passed (1)', false], +])('checks the native test summary: %s', (_label, output, expected) => { + expect(passedTests.test(stripVTControlCharacters(output))).toBe(expected) +}) it('integrates the configured plugin with the actual pinned Harness and Python MCP', () => { const root = resolve(dirname(fileURLToPath(import.meta.url)), '../..') @@ -14,5 +26,5 @@ it('integrates the configured plugin with the actual pinned Harness and Python M }) expect(result.error, 'Native host prerequisite/build missing; run repository install-all first').toBeUndefined() expect(result.status, result.stdout + result.stderr).toBe(0) - expect(result.stdout).toMatch(/5 passed/) + expect(stripVTControlCharacters(result.stdout)).toMatch(passedTests) }, 120000) diff --git a/packages/dsh-triton-riscv/tests/integration/python.spec.ts b/packages/dsh-triton-riscv/tests/integration/python.spec.ts new file mode 100644 index 0000000..b5ed6c6 --- /dev/null +++ b/packages/dsh-triton-riscv/tests/integration/python.spec.ts @@ -0,0 +1,58 @@ +import { beforeAll, describe, expect, it } from 'vitest' +import { spawnSync } from 'node:child_process' +import { existsSync, readdirSync } from 'node:fs' +import { dirname, join, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '../..') +const source = join(root, 'python') +const tests = join(source, 'codex_agent/tests') +const python = process.env.TRITON_PLUGIN_TEST_PYTHON || join(root, '.venv/bin/python') +const modules = readdirSync(tests) + .filter(name => /^test_.*\.py$/.test(name)) + .sort() + +beforeAll(() => { + expect(existsSync(python), 'Install the plugin backend with npm run setup:backend first').toBe(true) + for (const name of ['TRITON_MYSQL_URL', 'TRITON_TEST_REDIS_URL', 'TRITON_TEST_AMQP_URL']) { + expect(process.env[name], `Set ${name} to a disposable test service; see the plugin README`).toBeTruthy() + } + expect(modules.length, 'No Python test modules discovered').toBeGreaterThan(0) +}) + +// Pytest runs the existing TestCase and function-style tests; Vitest owns CI status. +describe.sequential('Python backend', () => { + for (const module of modules) { + it( + module, + () => { + const result = spawnSync( + python, + [ + '-I', + '-c', + 'import sys; sys.path.insert(0, sys.argv.pop(1)); import pytest; raise SystemExit(pytest.main(sys.argv[1:]))', + source, + join(tests, module), + '-q', + '-ra', + '--import-mode=importlib', + ], + { + cwd: source, + encoding: 'utf8', + timeout: 110_000, + maxBuffer: 16 * 1024 * 1024, + env: { ...process.env, PYTEST_DISABLE_PLUGIN_AUTOLOAD: '1', PYTEST_ADDOPTS: '' }, + }, + ) + const output = result.stdout + result.stderr + console.log(`${module}\n${output}`) + expect(result.error, 'Python process failed or timed out').toBeUndefined() + // Pytest also returns nonzero on collection errors or when no tests run. + expect(result.status, output).toBe(0) + }, + 120_000, + ) + } +}) diff --git a/packages/dsh-triton-riscv/tests/native-host.spec.ts b/packages/dsh-triton-riscv/tests/native-host.spec.ts index 2899632..0198f11 100644 --- a/packages/dsh-triton-riscv/tests/native-host.spec.ts +++ b/packages/dsh-triton-riscv/tests/native-host.spec.ts @@ -66,7 +66,6 @@ async function setup( ...process.env, TRITON_RISCV_REPO_ROOT: root, TRITON_RISCV_STATE_DIR: join(root, 'state'), - TRITON_RISCV_MEMORY_DB: join(root, 'state/memory.sqlite3'), TRITON_RISCV_MCP_PYTHON: python, TRITON_RISCV_ALLOW_DEVELOPMENT_APPLY: '1', TRITON_RISCV_ALLOW_VALIDATION: allowValidation ? '1' : '0', @@ -93,7 +92,6 @@ async function setup( if (options.noRepoRoot) { delete input.repoRoot delete input.stateDir - delete input.memory.database } const nativeFiber = await ctx.plugin(Native, input) const handles = new Map() @@ -140,7 +138,7 @@ async function setup( 'import os', 'from pathlib import Path', 'from codex_agent.memory import MemoryRecord, MemoryStore', - 'with MemoryStore(Path(os.environ["TRITON_RISCV_MEMORY_DB"])) as store:', + 'with MemoryStore(Path(os.environ["TRITON_RISCV_REPO_ROOT"])) as store:', ' store.add(MemoryRecord(memory_type="failure-diagnosis", operator="square_new", semantics="Compute the elementwise square of the input tensor.", pytorch_reference="torch.square(x)", summary="Synthetic native integration evidence; not a real validation result.", outcome="failed", confidence_grade="C", source_run="synthetic:native-integration", evidence={"recommended_actions":["Check the mask on the last block."]}))', ].join('\n'), ], @@ -213,7 +211,7 @@ it('real host guard denies generic bypass after activation, survives restore and it('real host + real stdio MCP: prepares, approves and applies in an isolated checkout', async () => { const h = await setup('allowed-once') - expect(h.ctx.tools.schemas(h.agent).filter(t => t.name.startsWith('mcp__triton_riscv__'))).toHaveLength(20) + expect(h.ctx.tools.schemas(h.agent).filter(t => t.name.startsWith('mcp__triton_riscv__'))).toHaveLength(21) const prepare = await h.call('prepare_operator_development', { specification: h.fixture.spec, }) @@ -460,7 +458,7 @@ it('starts without repoRoot, initializes once before first prompt, and exposes n const h = await setup('rejected', false, { noRepoRoot: true, initialize: false }) expect(h.ctx.tools.schemas(h.agent)).toHaveLength(0) const first = await h.ctx.systemPrompt.assemble(assembleContextFor(h.agent)) - expect(first.tools.filter(t => t.name.startsWith('mcp__triton_riscv__'))).toHaveLength(20) + expect(first.tools.filter(t => t.name.startsWith('mcp__triton_riscv__'))).toHaveLength(21) expect(h.ctx.tools.schemas()).toHaveLength(0) const again = await h.ctx.systemPrompt.assemble(assembleContextFor(h.agent)) expect(again.tools).toEqual(first.tools) @@ -498,7 +496,7 @@ it('two real workspace clients keep same-named sources, approvals and RAG eviden expect(contextA).toContain('synthetic:native-integration') expect(contextB).not.toContain('synthetic:native-integration') expect(contextB).not.toContain(idA) - expect(resolveSessionConfig(h.input, b.session).memory.database).not.toBe(h.env.TRITON_RISCV_MEMORY_DB) + expect(resolveSessionConfig(h.input, b.session).repoRoot).not.toBe(h.env.TRITON_RISCV_REPO_ROOT) const foreign = await h.call('apply_development_proposal', { proposal_id: idA }, b) expect(foreign.isError).toBe(true) expect(h.questions).toHaveLength(0) @@ -580,7 +578,7 @@ it('does not let a nested agent borrow its parent workspace connection or contex }) expect(before.isError).toBe(true) const assembly = await h.ctx.systemPrompt.assemble(assembleContextFor(child)) - expect(assembly.tools.filter(t => t.name.startsWith('mcp__triton_riscv__'))).toHaveLength(20) + expect(assembly.tools.filter(t => t.name.startsWith('mcp__triton_riscv__'))).toHaveLength(21) expect(renderContextSnapshot(assembly)).not.toContain('synthetic:native-integration') expect(data(await h.call('discover_operator', { operator_name: 'square_new' }, child)).status).toBe('not_found') }, 60_000) diff --git a/packages/dsh-triton-riscv/tests/unit/inputs.spec.ts b/packages/dsh-triton-riscv/tests/unit/inputs.spec.ts index f6df9b8..ac162ee 100644 --- a/packages/dsh-triton-riscv/tests/unit/inputs.spec.ts +++ b/packages/dsh-triton-riscv/tests/unit/inputs.spec.ts @@ -11,6 +11,15 @@ import { apply } from '../../native.js' import { apply as policy } from '../../index.js' describe('inputs: configuration and activation boundaries', () => { + it('keeps queue optional, typed and credentials outside settings', () => { + expect(resolveConfig().queue.enabled).toBe(false) + for (const queue of [{ enabled: 'yes' }, { leaseSeconds: 0 }, { urlEnv: 'TRITON_MYSQL_URL' }, { unknown: true }]) + expect(() => resolveConfig({ queue })).toThrow() + const config = resolveConfig({ repoRoot: '/tmp/op', queue: { enabled: true, urlEnv: 'MQ_SECRET' } }) + const env = workerEnvironment(config, { MQ_SECRET: 'amqp://temporary' }) + expect(env.MQ_SECRET).toBe('amqp://temporary') + expect(env[CONFIG_ENV]).not.toContain('amqp://temporary') + }) it('is inert until explicitly configured, without requiring Python or a checkout', async () => { const ctx = { plugin: vi.fn(), on: vi.fn(), effect: vi.fn() } await apply(ctx, {}) @@ -30,6 +39,12 @@ describe('inputs: configuration and activation boundaries', () => { { remote: { host: 'h;sh', repository: '/repo' } }, { remote: { host: 'host', repository: '/repo/../other' } }, { memory: { embedding: { tokenBudget: -1 } } }, + { memory: { database: '/old.sqlite3' } }, + { storage: { poolSize: 0 } }, + { storage: { maxOverflow: -1 } }, + { storage: { poolTimeout: 61 } }, + { storage: { urlEnv: 'PATH' } }, + { storage: { url: 'mysql+pymysql://user:secret@localhost/db' } }, { memory: { embedding: { apiKeyEnv: 'TRITON_RISCV_ALLOW_VALIDATION' } } }, ]) expect(() => resolveConfig(value)).toThrow() @@ -42,6 +57,24 @@ describe('inputs: configuration and activation boundaries', () => { expect(result.memory.contextFormat).toBe('classic') expect(result.memory.embedding.provider).toBe('none') expect(result).not.toHaveProperty('env') + expect(result.cache.enabled).toBe(false) + }) + it('validates Redis options and shares them without exposing credentials', () => { + for (const cache of [ + { enabled: 'true' }, + { urlEnv: 'PATH' }, + { lockMs: 0 }, + { unknown: 1 }, + { urlEnv: 'TRITON_MYSQL_URL' }, + { bloomErrorRate: 0 }, + { maxConcurrent: 0 }, + ]) + expect(() => resolveConfig({ cache })).toThrow() + const config = resolveConfig({ repoRoot: '/tmp/op', cache: { enabled: true, urlEnv: 'CACHE_SECRET' } }) + const env = workerEnvironment(config, { CACHE_SECRET: 'redis://:secret@localhost:6379/0' }) + expect(env.CACHE_SECRET).toContain('secret') + expect(env[CONFIG_ENV]).not.toContain('redis://') + expect(JSON.parse(env[CONFIG_ENV]).cache.enabled).toBe(true) }) it('uses the same explicit target and permissions for MCP and approval bridge', () => { const config = resolveConfig({ @@ -81,6 +114,24 @@ describe('inputs: configuration and activation boundaries', () => { expect(JSON.stringify(config)).not.toContain('secret-value') expect(mcpConfiguration(config, { EMBEDDING_TEST_KEY: 'secret-value' }).env.EMBEDDING_TEST_KEY).toBe('secret-value') }) + it('shares bounded MySQL settings but keeps the URL out of the configuration document', () => { + const config = resolveConfig({ + repoRoot: '/tmp/op', + storage: { urlEnv: 'WORKSPACE_DB_URL', poolSize: 2, maxOverflow: 0 }, + }) + const ambient = { WORKSPACE_DB_URL: 'mysql+pymysql://user:secret@localhost/db' } + const mcp = mcpConfiguration(config, ambient) + const bridge = bridgeEnvironment(config, ambient) + expect(mcp.env.WORKSPACE_DB_URL).toBe(ambient.WORKSPACE_DB_URL) + expect(bridge.WORKSPACE_DB_URL).toBe(ambient.WORKSPACE_DB_URL) + expect(mcp.env[CONFIG_ENV]).not.toContain('secret') + expect(JSON.parse(mcp.env[CONFIG_ENV]).storage).toEqual({ + urlEnv: 'WORKSPACE_DB_URL', + poolSize: 2, + maxOverflow: 0, + poolTimeout: 10, + }) + }) it('passes the strict resource requirement to both execution paths, never from ambient settings', () => { const config = resolveConfig({ repoRoot: '/tmp/op', diff --git a/packages/dsh-triton-riscv/tests/unit/local-environment.spec.ts b/packages/dsh-triton-riscv/tests/unit/local-environment.spec.ts index 87b01f0..837383f 100644 --- a/packages/dsh-triton-riscv/tests/unit/local-environment.spec.ts +++ b/packages/dsh-triton-riscv/tests/unit/local-environment.spec.ts @@ -13,7 +13,9 @@ function fixture(model?: string, local?: Record) { if (local !== undefined) writeFileSync(join(root, '.state/native-local.json'), JSON.stringify(local)) return root } -afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }) }) +afterEach(() => { + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }) +}) describe('shared local launcher environment', () => { it('preserves the existing environment when local files are absent', () => { @@ -23,47 +25,61 @@ describe('shared local launcher environment', () => { expect(result).not.toBe(ambient) }) it('loads existing target configuration without overriding shell settings', () => { - const result = loadLocalEnvironment(fixture(undefined, { - TRITON_RISCV_REPO_ROOT: '/operators', RISCV_HOST: 'sg2044', DSH_MODEL: 'saved-model', - }), { DSH_MODEL: 'shell-model' }) + const result = loadLocalEnvironment( + fixture(undefined, { + TRITON_RISCV_REPO_ROOT: '/operators', + RISCV_HOST: 'sg2044', + DSH_MODEL: 'saved-model', + }), + { DSH_MODEL: 'shell-model' }, + ) expect(result).toEqual({ TRITON_RISCV_REPO_ROOT: '/operators', RISCV_HOST: 'sg2044', DSH_MODEL: 'shell-model' }) }) it('uses the explicit model file instead of stale model and key exports', () => { const ambient = { DSH_MODEL: 'AZ-GPT-5.6-Sol', ISRC_API_KEY: 'old-test-key' } - const result = loadLocalEnvironment(fixture( - 'DSH_MODEL=gpt-6-astra\nISRC_BASE_URL=https://model.example/v1\nISRC_API_KEY="new-test-key"\n', - { DSH_MODEL: 'gpt-5.6-sol' }, - ), ambient) + const result = loadLocalEnvironment( + fixture('DSH_MODEL=gpt-6-astra\nISRC_BASE_URL=https://model.example/v1\nISRC_API_KEY="new-test-key"\n', { + DSH_MODEL: 'gpt-5.6-sol', + }), + ambient, + ) expect(result.DSH_MODEL).toBe('gpt-6-astra') expect(result.ISRC_API_KEY).toBe('new-test-key') expect(result.ISRC_BASE_URL).toBe('https://model.example/v1') expect(ambient.ISRC_API_KEY).toBe('old-test-key') }) it('supports comments and quoted values without executing shell expressions', () => { - const result = loadLocalEnvironment(fixture( - '# local values only\nDSH_MODEL="gpt-6-astra"\nISRC_API_KEY=literal-$(whoami)\n', - ), {}) + const result = loadLocalEnvironment( + fixture('# local values only\nDSH_MODEL="gpt-6-astra"\nISRC_API_KEY=literal-$(whoami)\n'), + {}, + ) expect(result.ISRC_API_KEY).toBe('literal-$(whoami)') }) it('rejects an empty key rather than silently reusing an old credential', () => { - expect(() => loadLocalEnvironment(fixture('ISRC_API_KEY=\n'), { ISRC_API_KEY: 'old-test-key' })) - .toThrow('Fill ISRC_API_KEY') + expect(() => loadLocalEnvironment(fixture('ISRC_API_KEY=\n'), { ISRC_API_KEY: 'old-test-key' })).toThrow( + 'Fill ISRC_API_KEY', + ) }) it('does not disclose malformed credentials in errors', () => { const secret = 'private test secret' let message = '' - try { loadLocalEnvironment(fixture(`ISRC_API_KEY="${secret}"\n`), {}) } - catch (error) { message = (error as Error).message } + try { + loadLocalEnvironment(fixture(`ISRC_API_KEY="${secret}"\n`), {}) + } catch (error) { + message = (error as Error).message + } expect(message).toContain('raw key only') expect(message).not.toContain(secret) }) it('rejects empty model names and unrelated permissions in the model file', () => { expect(() => loadLocalEnvironment(fixture('DSH_MODEL=\n'), {})).toThrow('Fill DSH_MODEL') - expect(() => loadLocalEnvironment(fixture('TRITON_RISCV_ALLOW_REPAIR_APPLY=1\n'), {})) - .toThrow('Unsupported setting') + expect(() => loadLocalEnvironment(fixture('TRITON_RISCV_ALLOW_REPAIR_APPLY=1\n'), {})).toThrow( + 'Unsupported setting', + ) }) it('still rejects credentials or unknown settings in native-local.json', () => { - expect(() => loadLocalEnvironment(fixture(undefined, { ISRC_API_KEY: 'test-only' }), {})) - .toThrow('Invalid local configuration key') + expect(() => loadLocalEnvironment(fixture(undefined, { ISRC_API_KEY: 'test-only' }), {})).toThrow( + 'Invalid local configuration key', + ) }) }) diff --git a/packages/dsh-triton-riscv/tests/unit/native-adapter.spec.ts b/packages/dsh-triton-riscv/tests/unit/native-adapter.spec.ts index 1ec4a60..a2d2fc4 100644 --- a/packages/dsh-triton-riscv/tests/unit/native-adapter.spec.ts +++ b/packages/dsh-triton-riscv/tests/unit/native-adapter.spec.ts @@ -189,13 +189,23 @@ async function plan(h, s, id = 'run-1') { } test('status queries require an owned plan and never reactivate an old plan or request approval', async () => { - const h = harness(), s = session('status-owner') + const h = harness(), + s = session('status-owner') await plan(h, s, 'old-plan') await plan(h, s, 'new-plan') const before = structuredClone(h.store.load(s.id)) - await h.call(s, 'get_validation_status', { run_id: 'old-plan', remote: true }, { - run_id: 'old-plan', operator: 'other', execution_state: 'unknown', remote_state: 'queued', test_status: null, - }) + await h.call( + s, + 'get_validation_status', + { run_id: 'old-plan', remote: true }, + { + run_id: 'old-plan', + operator: 'other', + execution_state: 'unknown', + remote_state: 'queued', + test_status: null, + }, + ) assert.deepEqual(h.store.load(s.id), before) assert.equal(h.questions.length, 0) const calls = h.calls() @@ -468,6 +478,33 @@ test('a real but superseded plan ID in the same session cannot execute', async ( assert.equal(h.questions.length, 1) }) +test('queued validation is not a receipt and only its owning session can collect it', async () => { + const h = harness(), + s = session('queued-owner') + await plan(h, s) + await h.call( + s, + 'execute_approved_validation', + { run_id: 'run-1' }, + { status: 'queued', job_id: 'async-1', run_id: 'run-1' }, + ) + assert.match(h.context(s), /queued/) + await assert.rejects(h.call(s, 'diagnose_failure', { run_id: 'run-1' }, {}), /not created|not the current/) + await assert.rejects(h.call(session('other'), 'inspect_queued_task', { job_id: 'async-1' }, {}), /not created/) + await h.call( + s, + 'inspect_queued_task', + { job_id: 'async-1' }, + { + kind: 'validation', + status: 'succeeded', + result: { status: 'passed', run_id: 'receipt-1', receipt_path: 'receipt.json' }, + }, + ) + assert.match(h.context(s), /receipt-1/) + assert.match(h.context(s), /passed/) +}) + test('a committed replay uses business checks without another approval dialog', async () => { const h = harness('allowed-once', async () => ({ replay: true, status: 'approved' })), s = session('s1') diff --git a/packages/dsh-triton-riscv/tests/unit/native-workspace.spec.ts b/packages/dsh-triton-riscv/tests/unit/native-workspace.spec.ts index 7e81564..fc35beb 100644 --- a/packages/dsh-triton-riscv/tests/unit/native-workspace.spec.ts +++ b/packages/dsh-triton-riscv/tests/unit/native-workspace.spec.ts @@ -32,26 +32,26 @@ it('selected workspace overrides the legacy fallback and binds every execution p expect(mcpConfiguration(config).cwd).toBe(b) expect(JSON.parse(bridgeEnvironment(config, { TRITON_RISCV_REPO_ROOT: a }).TRITON_RISCV_CONFIG).repoRoot).toBe(b) expect(config.stateDir).toBe(join(b, 'agent-results')) - expect(config.memory.database).toBe(join(b, 'agent-results/memory.sqlite3')) + expect(config.storage.urlEnv).toBe('TRITON_MYSQL_URL') }) -it('preserves existing explicit state/database paths only for the legacy target', () => { +it('preserves explicit state paths only for the legacy target', () => { const { root, a, b } = fixture() - const input = { repoRoot: a, stateDir: join(root, 'state'), memory: { database: join(root, 'old.sqlite3') } } + const input = { repoRoot: a, stateDir: join(root, 'state') } const old = resolveSessionConfig(input, session())! expect(old.repoRoot).toBe(a) expect(old.stateDir).toBe(input.stateDir) - expect(old.memory.database).toBe(input.memory.database) const changed = resolveSessionConfig(input, session(b))! expect(changed.stateDir).toContain('/workspaces/') - expect(changed.memory.database).not.toBe(old.memory.database) + expect(changed.repoRoot).not.toBe(old.repoRoot) }) it('namespaces custom storage across workspaces and reuses it across sessions in the same workspace', () => { const { root, a, b } = fixture() - const input = { stateDir: join(root, 'state'), memory: { database: join(root, 'memory.sqlite3') } } + const input = { stateDir: join(root, 'state') } const one = resolveSessionConfig(input, session(a))!, two = resolveSessionConfig(input, session(b))! expect(one.stateDir).not.toBe(two.stateDir) - expect(one.memory.database).not.toBe(two.memory.database) + expect(one.repoRoot).not.toBe(two.repoRoot) + expect(one.storage).toEqual(two.storage) expect(resolveSessionConfig(input, { ...session(a), id: 'another' })).toEqual(one) }) it('canonicalizes symlinks so aliases share the same repository history', () => { diff --git a/packages/dsh-triton-riscv/vitest.config.ts b/packages/dsh-triton-riscv/vitest.config.ts index 76c399e..43cfd8a 100644 --- a/packages/dsh-triton-riscv/vitest.config.ts +++ b/packages/dsh-triton-riscv/vitest.config.ts @@ -1,5 +1,5 @@ import { defineConfig } from 'vitest/config' export default defineConfig({ - test: { include: ['tests/unit/**/*.spec.ts'], environment: 'node' }, + test: { include: ['tests/unit/**/*.spec.ts', 'tests/integration/**/*.spec.ts'], environment: 'node' }, })