Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions packages/mcp-core/src/api-client/schema.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1353,6 +1353,7 @@ export const AutofixRunStateSchema = z.object({
run_id: z.number(),
updated_at: z.string().nullable().optional(),
status: AutofixStatusSchema,
failure_reason: z.string().nullable().optional(),
blocks: z.array(AutofixBlockSchema).default([]),
pending_user_input: z.unknown().nullable().optional(),
repo_pr_states: z
Expand Down
4 changes: 4 additions & 0 deletions packages/mcp-core/src/internal/formatting.ts
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ import {
getAutofixArtifactSummaries,
getStatusDisplayName,
isTerminalStatus,
SEER_CONTINUE_OUT_OF_CREDITS_MESSAGE,
} from "./tool-helpers/seer";
import { formatToolCallInstruction } from "./tool-helpers/tool-call-formatting";
import { isPlainObject } from "./type-guards";
Expand Down Expand Up @@ -2200,6 +2201,9 @@ function formatSeerSummary(autofixState: AutofixRunState | undefined): string {
if (autofix.status === "error") {
parts.push("");
parts.push("**Status:** Analysis failed.");
if (autofix.failure_reason === "out_of_credits") {
parts.push(SEER_CONTINUE_OUT_OF_CREDITS_MESSAGE);
}
} else if (autofix.status === "awaiting_user_input") {
parts.push("");
parts.push("**Status:** Analysis paused - additional information needed.");
Expand Down
34 changes: 34 additions & 0 deletions packages/mcp-core/src/internal/tool-helpers/seer.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import { z } from "zod";
import type { AutofixRunState } from "../../api-client/index";
import type { Issue } from "../../api-client/types";
import { isPlainObject } from "../type-guards";

type AutofixRun = NonNullable<AutofixRunState["autofix"]>;

Expand Down Expand Up @@ -85,6 +86,39 @@ export function getHumanInterventionGuidance(status: string): string {
return "";
}

export const SEER_START_OUT_OF_CREDITS_MESSAGE =
"Your organization does not have enough credits to start an Autofix run. Someone with billing access can turn on auto-reload in Sentry.";

export const SEER_CONTINUE_OUT_OF_CREDITS_MESSAGE =
"Your organization does not have enough credits to continue your Autofix run. Someone with billing access can turn on auto-reload in Sentry.";

const SEER_NO_QUOTA_MESSAGE =
"Your organization does not have enough Seer quota to start an Autofix run.";

/**
* Explain a 402 from starting a run. Sentry sends the out-of-credits 402 as
* `{ detail }` and the out-of-quota 402 as a bare string, which the API client
* can't parse.
*/
export function getSeerPaymentRequiredMessage(responseBody: unknown): string {
if (isPlainObject(responseBody) && typeof responseBody.detail === "string") {
return SEER_START_OUT_OF_CREDITS_MESSAGE;
}
return SEER_NO_QUOTA_MESSAGE;
}

/**
* Explain why a failed run stopped, when Seer gives a reason we can explain.
*/
export function getFailureReasonGuidance(
failureReason: string | null | undefined,
): string {
if (failureReason === "out_of_credits") {
return `\n${SEER_CONTINUE_OUT_OF_CREDITS_MESSAGE}\n`;
}
return "";
}

function escapeXmlAttribute(value: string | number): string {
return String(value)
.replace(/&/g, "&amp;")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -439,6 +439,97 @@ describe("analyze_issue_with_seer", () => {
},
);

it.each([
[
"out of credits",
{
detail:
"Your organization does not have enough credits to start an Autofix run.",
},
"Your organization does not have enough credits to start an Autofix run. Someone with billing access can turn on auto-reload in Sentry.",
],
[
"out of quota",
"No budget for Seer Autofix.",
"Your organization does not have enough Seer quota to start an Autofix run.",
],
])(
"explains why the run did not start when Sentry answers 402 (%s)",
async (_label, body, expected) => {
let getCallCount = 0;

mswServer.use(
http.get(
"https://sentry.io/api/0/organizations/sentry-mcp-evals/issues/CLOUDFLARE-MCP-402/autofix/",
() => {
getCallCount++;
return HttpResponse.json({ autofix: null });
},
),
http.post(
"https://sentry.io/api/0/organizations/sentry-mcp-evals/issues/CLOUDFLARE-MCP-402/autofix/",
() => HttpResponse.json(body, { status: 402 }),
),
);

const result = await analyzeIssueWithSeer.handler(
{
organizationSlug: "sentry-mcp-evals",
regionUrl: null,
issueId: "CLOUDFLARE-MCP-402",
instruction: undefined,
},
{
constraints: { organizationSlug: undefined },
accessToken: "access-token",
userId: "1",
},
);

expect(result).toBe(
`# Seer Analysis for Issue CLOUDFLARE-MCP-402\n\n## Analysis Not Started\n\n${expected}\n`,
);
// No polling after a refused start.
expect(getCallCount).toBe(1);
},
);

it("explains a run that stopped because the org ran out of credits", async () => {
mswServer.use(
http.get(
"https://sentry.io/api/0/organizations/sentry-mcp-evals/issues/CLOUDFLARE-MCP-CREDITS/autofix/",
() =>
HttpResponse.json({
autofix: {
run_id: 456,
status: "error",
failure_reason: "out_of_credits",
blocks: [],
},
}),
),
);

const result = await analyzeIssueWithSeer.handler(
{
organizationSlug: "sentry-mcp-evals",
regionUrl: null,
issueId: "CLOUDFLARE-MCP-CREDITS",
instruction: undefined,
},
{
constraints: { organizationSlug: undefined },
accessToken: "access-token",
userId: "1",
},
);

expect(result).toContain("## Analysis Failed");
expect(result).toContain(
"Your organization does not have enough credits to continue your Autofix run. Someone with billing access can turn on auto-reload in Sentry.",
);
});

it("returns unsupported message for metric issues without calling Seer", async () => {
let autofixRequests = 0;

Expand Down
29 changes: 24 additions & 5 deletions packages/mcp-core/src/tools/catalog/analyze-issue-with-seer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,10 @@ import {
} from "../../internal/tool-helpers/issue";
import {
getActiveAutofixTodo,
getFailureReasonGuidance,
getHumanInterventionGuidance,
getOutputForAutofixRun,
getSeerPaymentRequiredMessage,
getSeerUnsupportedIssueMessage,
getStatusDisplayName,
isSeerSupportedIssue,
Expand Down Expand Up @@ -141,12 +143,23 @@ export default defineTool({

// Step 2: Start analysis if none exists
if (!autofixState.autofix) {
let startResult: Awaited<ReturnType<typeof apiService.startAutofix>>;
try {
startResult = await apiService.startAutofix({
organizationSlug: orgSlug,
issueId: parsedIssueId,
instruction: params.instruction,
});
} catch (error) {
// Sentry answers 402 before starting a run when the org is out of
// Seer credits or quota.
if (error instanceof ApiError && error.status === 402) {
output += `## Analysis Not Started\n\n${getSeerPaymentRequiredMessage(error.responseBody)}\n`;
return output;
}
throw error;
}
output += `Starting new analysis...\n\n`;
const startResult = await apiService.startAutofix({
organizationSlug: orgSlug,
issueId: parsedIssueId,
instruction: params.instruction,
});
output += `Analysis started with Run ID: ${startResult.run_id}\n\n`;

// Give it a moment to initialize
Expand Down Expand Up @@ -182,6 +195,9 @@ export default defineTool({

if (existingStatus !== "completed") {
output += `\n**Status**: ${existingStatus}\n`;
output += getFailureReasonGuidance(
autofixState.autofix.failure_reason,
);
output += getHumanInterventionGuidance(existingStatus);
output += "\n";
}
Expand Down Expand Up @@ -215,6 +231,9 @@ export default defineTool({

if (status !== "completed") {
output += `\n**Status**: ${status}\n`;
output += getFailureReasonGuidance(
autofixState.autofix.failure_reason,
);
output += getHumanInterventionGuidance(status);
}

Expand Down
Loading