plugin.js

// AI Agent Evals for PromptOps. Section: pages.
//
// The plugin never talks to a model. It picks models from the list PromptOps gives it and asks for a
// measurement. PromptOps writes the prompt, runs the CLIs and answers with numbers only.
// The page is described as plain data and PromptOps draws it.
(() => {
  'use strict';

  const PAGE = 'evals';
  const state = { models: [], selected: {}, running: false, progress: null, results: [], notice: null, loaded: false };

  const num = (v) => (typeof v === 'number' && isFinite(v) ? v : null);
  const text = (v, max) => String(v == null ? '' : v).slice(0, max);
  const tps = (v) => (num(v) === null ? '—' : (Math.round(v * 10) / 10).toFixed(1) + ' t/s');
  const band = (b) => (b && b.expectedTps ? Math.round(b.expectedTps.min) + '–' + Math.round(b.expectedTps.max) + ' t/s' : '');

  /** What PromptOps returns is data from outside: keep only the shape this page needs. */
  function cleanModels(raw) {
    const list = raw && Array.isArray(raw.data) ? raw.data : [];
    return list.slice(0, 40).map((m) => ({
      id: text(m.id, 80), label: text(m.label, 80), provider: text(m.provider, 40), providerLabel: text(m.providerLabel, 40),
      available: m.available !== false, baseline: m.baseline || null,
      history: { runs: (m.history && num(m.history.runs)) || 0, medianTps: m.history ? num(m.history.medianTps) : null, lastRunAt: m.history ? text(m.history.lastRunAt, 40) : '' },
    })).filter((m) => m.id);
  }

  function cleanResults(raw) {
    return (Array.isArray(raw) ? raw : []).slice(0, 20).map((r) => ({
      id: text(r.id, 80), label: text(r.label || r.id, 80), providerLabel: text(r.providerLabel, 40), ok: r.ok === true, error: text(r.error, 300),
      tokensPerSec: num(r.tokensPerSec), outputTokens: num(r.outputTokens), tokensEstimated: r.tokensEstimated === true, sampleMs: num(r.sampleMs),
      classification: ['below', 'within', 'above'].includes(r.classification) ? r.classification : 'unknown', baseline: r.baseline || null,
      historyMedian: r.history ? num(r.history.medianTps) : null, historyRuns: (r.history && num(r.history.runs)) || 0,
    }));
  }

  async function loadModels(sdk) {
    state.models = cleanModels(await sdk.ai.benchmarkModels());
    state.loaded = true;
  }

  const selectedIds = () => state.models.filter((m) => m.available && state.selected[m.id]).map((m) => m.id);

  // ── Reading a result ─────────────────────────────────────────────────────────

  function versusReference(r) {
    const range = band(r.baseline);
    if (!range || r.classification === 'unknown') return '—';
    return { below: 'Slower than ' + range, within: 'Within ' + range, above: 'Faster than ' + range }[r.classification];
  }

  /** Against your own machine: the median of your earlier runs. One run alone says nothing yet. */
  function versusUsual(r) {
    if (r.historyMedian === null || r.historyRuns < 2 || r.tokensPerSec === null) return 'First runs';
    const delta = Math.round(((r.tokensPerSec - r.historyMedian) / r.historyMedian) * 100);
    if (Math.abs(delta) < 10) return 'As usual';
    return (delta > 0 ? '+' : '−') + Math.abs(delta) + '% vs your ' + tps(r.historyMedian);
  }

  function resultsTable(results) {
    const ranked = results.filter((r) => r.ok).sort((a, b) => b.tokensPerSec - a.tokensPerSec).concat(results.filter((r) => !r.ok));
    return { type: 'table', columns: ['Model', 'Speed', 'Expected', 'Your usual', 'Tokens', 'Seconds'],
      rows: ranked.map((r) => (r.ok
        ? [r.label, tps(r.tokensPerSec), versusReference(r), versusUsual(r), (r.tokensEstimated ? '~' : '') + r.outputTokens, (r.sampleMs / 1000).toFixed(1)]
        : [r.label, 'Failed', '—', '—', '—', '—'])) };
  }

  // ── The page, as data ───────────────────────────────────────────────────────

  function view() {
    const left = [{ type: 'text', muted: true, text: 'Each model gets the same short writing task, three times, through the CLI you already use. The speed is the tokens it reports divided by the whole time, CLI start included. The middle run counts.' }];
    const providers = [];
    for (const m of state.models) if (!providers.includes(m.provider)) providers.push(m.provider);
    for (const provider of providers) {
      const group = state.models.filter((m) => m.provider === provider);
      left.push({ type: 'heading', text: group[0].providerLabel });
      if (!group[0].available) {
        left.push({ type: 'notice', tone: 'info', text: group[0].providerLabel + ' is not installed on this computer, so its models cannot be measured.' });
        continue;
      }
      for (const m of group) {
        left.push({ type: 'field', kind: 'toggle', id: 'm_' + state.models.indexOf(m), half: true, value: !!state.selected[m.id],
          label: m.label + (band(m.baseline) ? ' · ' + band(m.baseline) : '') });
      }
    }
    if (!state.models.length) left.push({ type: 'notice', tone: 'warning', text: 'PromptOps returned no models to measure.' });
    const count = selectedIds().length;
    left.push({ type: 'actions', items: [
      { id: 'run', label: state.running ? 'Measuring…' : 'Run speed test', style: 'primary', icon: 'ti-gauge', disabled: state.running || !state.models.length },
      { id: 'all', label: 'Select all', disabled: state.running }, { id: 'none', label: 'Clear', style: 'ghost', disabled: state.running },
    ] });
    left.push({ type: 'text', muted: true, text: count ? count + ' selected: ' + count * 3 + ' calls on your accounts. You confirm before anything runs.' : 'Pick at least one model.' });

    const right = [];
    if (state.notice) right.push({ type: 'notice', tone: state.notice.tone, text: state.notice.text });
    if (state.running && state.progress) {
      const p = state.progress;
      right.push({ type: 'progress', value: p.total ? (p.index + (p.samples ? p.sample / p.samples : 0)) / p.total : 0,
        label: p.done ? 'Finishing…' : 'Measuring ' + p.label + ' · run ' + Math.min(p.sample + 1, p.samples) + ' of ' + p.samples + ' · model ' + (p.index + 1) + ' of ' + p.total });
    }
    if (state.results.length) {
      const good = state.results.filter((r) => r.ok);
      if (good.length) {
        const fastest = good.slice().sort((a, b) => b.tokensPerSec - a.tokensPerSec)[0];
        right.push({ type: 'stats', items: [
          { label: 'Fastest', value: fastest.label, hint: tps(fastest.tokensPerSec) }, { label: 'Its speed', value: tps(fastest.tokensPerSec) },
          { label: 'Measured', value: good.length + ' of ' + state.results.length },
        ] });
      }
      right.push(resultsTable(state.results));
      for (const r of state.results.filter((x) => !x.ok)) right.push({ type: 'notice', tone: 'warning', text: r.label + ': ' + (r.error || 'no measurement') });
      if (good.some((r) => r.tokensEstimated)) right.push({ type: 'text', muted: true, text: '~ means the provider did not report its tokens, so they are estimated from the length of the answer. Those runs stay out of your history.' });
    } else if (!state.running) {
      const known = state.models.filter((m) => m.history.runs > 0);
      if (known.length) {
        right.push({ type: 'heading', text: 'Your history on this computer' });
        right.push({ type: 'table', columns: ['Model', 'Your usual', 'Runs', 'Expected'],
          rows: known.map((m) => [m.label, tps(m.history.medianTps), String(m.history.runs), band(m.baseline) || '—']) });
      } else {
        right.push({ type: 'text', muted: true, text: 'Your results appear here. After a few runs you also see how today compares with what is normal on this computer.' });
      }
    }
    return { title: 'AI Agent Evals', subtitle: 'How fast your models really answer, measured on your own accounts. PromptOps runs the test: this plugin only reads the numbers.',
      blocks: [{ type: 'columns', columns: [left, right] }] };
  }

  function absorb(values) {
    if (!values || typeof values !== 'object') return;
    state.models.forEach((m, i) => { if (('m_' + i) in values) state.selected[m.id] = values['m_' + i] === true && m.available; });
  }

  // PromptOps reports every finished run. The page follows along.
  promptops.events.on('benchmark.progress', async (payload, sdk) => {
    if (!state.running || !payload) return;
    state.progress = { index: num(payload.index) || 0, total: num(payload.total) || 0, sample: num(payload.sample) || 0, samples: num(payload.samples) || 0,
      label: text(payload.label, 80), done: payload.done === true };
    state.results = cleanResults(payload.results);
    try { await sdk.pages.update(PAGE, view()); } catch (e) { /* the page may be closed */ }
  });

  promptops.pages.register({
    async open(pageId, _ctx, sdk) {
      if (pageId !== PAGE) throw new Error('unknown page');
      state.notice = null;
      try {
        await loadModels(sdk);
        const saved = await sdk.storage.get('selected');
        if (Array.isArray(saved) && !Object.keys(state.selected).length) for (const id of saved) state.selected[text(id, 80)] = true;
      } catch (e) {
        state.notice = { tone: 'danger', text: String(e && e.message || e) };
      }
      return view();
    },

    async event(pageId, ev, sdk) {
      if (pageId !== PAGE) throw new Error('unknown page');
      absorb(ev && ev.values);
      state.notice = null;
      const id = ev && ev.type === 'action' ? ev.id : null;
      if (id === 'all') state.models.forEach((m) => { state.selected[m.id] = m.available; });
      if (id === 'none') state.selected = {};
      if (id !== 'run') return view();

      const models = selectedIds();
      if (!models.length) { state.notice = { tone: 'warning', text: 'Pick at least one model first.' }; return view(); }
      state.running = true; state.results = []; state.progress = null;
      try {
        try { await sdk.storage.set('selected', models); } catch (e) { /* a convenience */ }
        const res = await sdk.ai.benchmark(models);
        state.results = cleanResults(res && res.results);
        await loadModels(sdk);
      } catch (e) {
        const message = String(e && e.message || e);
        state.notice = /^(user_denied|confirm_timeout)\b/.test(message) ? { tone: 'info', text: 'Cancelled. Nothing was run.' } : { tone: 'danger', text: message };
      } finally {
        state.running = false; state.progress = null;
      }
      return view();
    },
  });
})();