From 828055361c31c7b3458cf680474351fce941eaed Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Wed, 7 Oct 2026 13:54:46 -0700 Subject: [PATCH 1/4] List every scope Google accepts on each hosted Google operation --- scripts/generate-google-specs.ts | 22 +++- scripts/google-specs.ts | 182 ++++++++++++++++++++++++++++++- 2 files changed, 199 insertions(+), 5 deletions(-) diff --git a/scripts/generate-google-specs.ts b/scripts/generate-google-specs.ts index 218c96d5..6df71612 100644 --- a/scripts/generate-google-specs.ts +++ b/scripts/generate-google-specs.ts @@ -9,7 +9,8 @@ * converter — the exact pipeline the product executes at add time — and then * corrects what that converter's multi-service bundle output gets wrong for a * document describing one service (scripts/google-specs.ts): realGooglePaths - * keys every operation by its real path, and hostedGoogleSpec fixes the + * keys every operation by its real path, googleOperationScopes lists every + * scope Google accepts for each operation, and hostedGoogleSpec fixes the * document server, types Discovery's string-encoded defaults, writes everything * in a fixed order and refuses any operation left on a placeholder path. * @@ -28,6 +29,7 @@ import { existsSync, mkdirSync, writeFileSync } from "node:fs"; import { join, resolve, dirname } from "node:path"; import { fileURLToPath } from "node:url"; import { + googleOperationScopes, hostedGoogleSpec, isJsonObject, parseJsonObject, @@ -51,6 +53,17 @@ const { googleOpenApiPresets, googleCatalogOAuthScopesForPreset } = await import const { convertGoogleDiscoveryBundleToOpenApi, normalizeGoogleDiscoveryUrl } = await import( join(providerDir, "discovery.ts") ); +const { compactGoogleOAuthScopes, isGoogleUserConsentOAuthScope } = await import( + join(providerDir, "oauth-scopes.ts") +); + +/** Executor's rule for whether a consent scope grants a method scope, as its + * converter applies it (googleScopeCovers in discovery.ts, not exported). */ +const scopeCovers = (consent: string, scope: string): boolean => + consent === scope || + (isGoogleUserConsentOAuthScope(scope) && + !compactGoogleOAuthScopes([consent, scope]).includes(scope)); + // The converter returns executor-flavored Effects; run them with executor's // own effect instance so the runtime identities match. const { Effect } = await import(join(EXECUTOR, "node_modules/effect/dist/index.js")); @@ -109,7 +122,12 @@ async function main(): Promise { parseJsonObject(documentText, `${preset.id} Discovery document ${index}`), ); const hosted = hostedGoogleSpec( - realGooglePaths(converted, discoveryDocuments, preset.id), + googleOperationScopes( + realGooglePaths(converted, discoveryDocuments, preset.id), + discoveryDocuments, + scopeCovers, + preset.id, + ), preset.id, ); if (hosted === undefined) { diff --git a/scripts/google-specs.ts b/scripts/google-specs.ts index 129c6425..b0e781ce 100644 --- a/scripts/google-specs.ts +++ b/scripts/google-specs.ts @@ -21,6 +21,9 @@ * uses. * - It copies Discovery defaults verbatim, and Discovery encodes every default * as a string, so a boolean parameter declares `default: "true"`. + * - It gives each operation only the consent scopes that cover its method, so + * every Gmail operation, reads included, declares full mail access + * (`https://mail.google.com/`) although Google accepts `gmail.readonly`. * - It writes paths, schemas, properties and parameters in Discovery's order, * which changes on every fetch. * @@ -28,7 +31,9 @@ * method and template no other operation shares keeps that template and its * resource-name parameter, such as `name: "spaces/AAA/messages/BBB"`. * Operations that share one are told apart by their parameters' Discovery - * patterns. It needs the Discovery documents, so only generation runs it. + * patterns. `googleOperationScopes` lists every scope Discovery accepts for + * each operation that its converted scopes cover. Both need the Discovery + * documents, so only generation runs them. * `hostedGoogleSpec` rewrites the rest and refuses an operation whose path is * not real. Generation and the committed-spec test both run `hostedGoogleSpec`, * so a regenerated spec cannot reintroduce any of these mistakes. @@ -343,6 +348,147 @@ export function realGooglePaths( return { ...spec, paths }; } +// --------------------------------------------------------------------------- +// Operation scopes +// --------------------------------------------------------------------------- + +const OAUTH_SCHEME = "googleOAuth2"; +const GOOGLE_SCOPES = "x-google-scopes"; + +/** Whether a token granted `consent` may call a method that accepts `scope`. + * Generation passes executor's rule, the one its converter applied. */ +export type ScopeCovers = (consent: string, scope: string) => boolean; + +/** The scopes an operation's security requirements name. */ +function securityScopes(security: Json | undefined, at: string): readonly string[] { + if (security === undefined) return []; + return arrayAt(security, at).flatMap((requirement) => + Object.values(objectAt(requirement, at)).flatMap((scopes) => + arrayAt(scopes, at).map((scope) => { + if (typeof scope !== "string") throw new Error(`${at}: expected scope names`); + return scope; + }), + ), + ); +} + +/** The scope descriptions the Discovery documents declare. */ +function discoveryScopeDescriptions(discovery: readonly JsonObject[]): ReadonlyMap { + const descriptions = new Map(); + for (const document of discovery) { + const auth = isJsonObject(document.auth) ? document.auth.oauth2 : undefined; + const scopes = isJsonObject(auth) ? auth.scopes : undefined; + if (isJsonObject(scopes)) + for (const [scope, value] of Object.entries(scopes)) + if (isJsonObject(value) && typeof value.description === "string") + descriptions.set(scope, value.description); + } + return descriptions; +} + +/** + * Lists on each operation every scope Google accepts for it, not only the + * broad scope the integration asks consent for. + * + * Executor's converter gives each operation the consent scopes that cover its + * method, so every Gmail operation, reads included, declared only + * `https://mail.google.com/`. Discovery lists the scopes each method accepts + * (`users.messages.list` accepts `gmail.readonly`, `gmail.metadata`, + * `gmail.modify` and full access). Each operation now names, as alternative + * security requirements, every scope Discovery lists for its method that one + * of its converted scopes covers, narrower scopes first, then the converted + * scopes. The covered rule keeps scopes Google will not grant through user + * consent out. An operation with no Discovery method or no Discovery scopes + * (executor's own Photos upload, Photos Picker's unscoped methods) keeps the + * scopes the converter gave it. The OAuth scheme declares every scope an + * operation names, with Discovery's description. Run after `realGooglePaths`. + * The input is not modified. + */ +export function googleOperationScopes( + spec: JsonObject, + discovery: readonly JsonObject[], + covers: ScopeCovers, + at: string, +): JsonObject { + const methods = discoveryMethods(discovery, at); + const descriptions = discoveryScopeDescriptions(discovery); + const used = new Set(); + const paths = Object.fromEntries( + Object.entries(objectAt(spec.paths, `${at} paths`)).map(([path, value]) => { + const item = objectAt(value, `${at} ${path}`); + return [ + path, + Object.fromEntries( + Object.entries(item).map(([method, field]): [string, Json] => { + if (!HTTP_METHODS.has(method)) return [method, field]; + const where = `${at} ${method} ${path}`; + const operation = objectAt(field, where); + const granted = securityScopes(operation.security, `${where} security`); + granted.forEach((scope) => used.add(scope)); + const id = operation.operationId; + const discoveryMethod = + typeof id !== "string" + ? undefined + : (methods.get(id) ?? + (id.endsWith("Media") ? methods.get(id.slice(0, -"Media".length)) : undefined)); + const accepted = discoveryMethod?.scopes; + if (accepted === undefined || granted.length === 0) return [method, operation]; + const narrower = arrayAt(accepted, `${where} Discovery scopes`) + .flatMap((scope) => { + if (typeof scope !== "string") + throw new Error(`${where}: Discovery scopes are not names`); + return granted.includes(scope) || + !granted.some((consent) => covers(consent, scope)) + ? [] + : [scope]; + }) + .sort(byText); + const scopes = [...narrower, ...granted]; + scopes.forEach((scope) => used.add(scope)); + return [ + method, + { + ...operation, + security: scopes.map((scope) => ({ [OAUTH_SCHEME]: [scope] })), + [GOOGLE_SCOPES]: scopes, + }, + ]; + }), + ), + ]; + }), + ); + + const components = objectAt(spec.components, `${at} components`); + const schemes = objectAt(components.securitySchemes, `${at} securitySchemes`); + const scheme = objectAt(schemes[OAUTH_SCHEME], `${at} ${OAUTH_SCHEME}`); + const flows = objectAt(scheme.flows, `${at} ${OAUTH_SCHEME} flows`); + const flow = objectAt(flows.authorizationCode, `${at} ${OAUTH_SCHEME} authorizationCode`); + const declared = objectAt(flow.scopes, `${at} ${OAUTH_SCHEME} scopes`); + const scopes: JsonObject = { ...declared }; + for (const scope of used) { + const description = declared[scope]; + scopes[scope] = + typeof description === "string" && description !== "" + ? description + : (descriptions.get(scope) ?? ""); + } + return { + ...spec, + paths, + components: { + ...components, + securitySchemes: { + ...schemes, + [OAUTH_SCHEME]: { + ...scheme, + flows: { ...flows, authorizationCode: { ...flow, scopes } }, + }, + }, + }, + }; +} + // --------------------------------------------------------------------------- // Hosted form // --------------------------------------------------------------------------- @@ -369,13 +515,15 @@ export interface HostedGoogleSpec { * - Paths, methods, schemas, properties and parameters are written in a fixed * order, so the same service always produces the same document. * - * @throws when an operation is not keyed by a real path, when the operations - * span more than one origin, or when a default cannot be read as its schema's + * @throws when an operation is not keyed by a real path, when a security + * requirement names a scope its OAuth scheme does not declare, when the + * operations span more than one origin, or when a default cannot be read as its schema's * type. Each is a change in Google's documents or executor's converter that a * person must look at, not something to publish. */ export function hostedGoogleSpec(spec: JsonObject, at: string): HostedGoogleSpec | undefined { assertRealPaths(spec, at); + assertDeclaredScopes(spec, at); const routed = withOperationServer(spec, at); if (routed === undefined) return undefined; const schemas = withSchemas(routed.document, at, (schema, where) => @@ -435,6 +583,34 @@ function assertRealPaths(spec: JsonObject, at: string): void { } } +/** @throws when a security requirement names a scope its OAuth scheme does not declare. */ +function assertDeclaredScopes(spec: JsonObject, at: string): void { + const components = isJsonObject(spec.components) ? spec.components : {}; + const schemes = isJsonObject(components.securitySchemes) ? components.securitySchemes : {}; + const declared = (name: string, where: string): JsonObject | undefined => { + const scheme = schemes[name]; + if (!isJsonObject(scheme)) throw new Error(`${where}: no security scheme ${name}`); + if (scheme.type !== "oauth2") return undefined; + const flows = objectAt(scheme.flows, `${at} ${name} flows`); + return objectAt(objectAt(flows.authorizationCode, `${at} ${name} authorizationCode`).scopes, `${at} ${name} scopes`); + }; + const check = (security: Json | undefined, where: string): void => { + if (security === undefined) return; + for (const requirement of arrayAt(security, `${where} security`)) + for (const [name, scopes] of Object.entries(objectAt(requirement, `${where} security`))) { + const known = declared(name, where); + if (known === undefined) continue; + for (const scope of arrayAt(scopes, `${where} security`)) + if (typeof scope !== "string" || !(scope in known)) + throw new Error(`${where}: ${name} does not declare the scope ${JSON.stringify(scope)}`); + } + }; + check(spec.security, at); + for (const [path, value] of Object.entries(objectAt(spec.paths, `${at} paths`))) + for (const { method, operation } of itemOperations(objectAt(value, `${at} ${path}`), `${at} ${path}`)) + check(operation.security, `${at} ${method} ${path}`); +} + function singleServerUrl(servers: Json | undefined, at: string): string | undefined { if (servers === undefined) return undefined; if (!Array.isArray(servers) || servers.length !== 1) From af5af4adacf5b736f6493bcda98ae60fdb70289d Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Wed, 7 Oct 2026 14:11:45 -0700 Subject: [PATCH 2/4] List every user-consent Discovery scope on each Google operation, not only covered ones --- scripts/generate-google-specs.ts | 46 ++++++-- scripts/google-specs.ts | 178 +++++++++++++++++++++++++------ 2 files changed, 183 insertions(+), 41 deletions(-) diff --git a/scripts/generate-google-specs.ts b/scripts/generate-google-specs.ts index 6df71612..0b25a4af 100644 --- a/scripts/generate-google-specs.ts +++ b/scripts/generate-google-specs.ts @@ -34,6 +34,8 @@ import { isJsonObject, parseJsonObject, realGooglePaths, + unlistedDiscoveryScopes, + type GoogleScopeRules, } from "./google-specs.ts"; const ROOT = join(dirname(fileURLToPath(import.meta.url)), ".."); @@ -56,13 +58,35 @@ const { convertGoogleDiscoveryBundleToOpenApi, normalizeGoogleDiscoveryUrl } = a const { compactGoogleOAuthScopes, isGoogleUserConsentOAuthScope } = await import( join(providerDir, "oauth-scopes.ts") ); +const { googleDiscoveryPolicyFor } = await import(join(providerDir, "service-policy.ts")); -/** Executor's rule for whether a consent scope grants a method scope, as its - * converter applies it (googleScopeCovers in discovery.ts, not exported). */ -const scopeCovers = (consent: string, scope: string): boolean => - consent === scope || - (isGoogleUserConsentOAuthScope(scope) && - !compactGoogleOAuthScopes([consent, scope]).includes(scope)); +/** Executor's Google scope rules, as its converter applies them. Neither + * discoveryMethodScopesForService nor googleScopeCovers in discovery.ts is + * exported, so these repeat them over the exported parts. One difference: + * compaction renames `userinfo.email` to `email` rather than dropping it, so + * googleScopeCovers has every scope cover the identity scopes, and each + * identity scope cover the other. Here a scope covers another only when + * compaction drops the other and keeps it unchanged. */ +const scopeRules: GoogleScopeRules = { + methodScopes: (service, version, scopes) => { + const policy = googleDiscoveryPolicyFor(service, version) as + | { + readonly authoritativeScopes?: Readonly>; + readonly fallbackMethodScopes?: readonly string[]; + } + | undefined; + if (scopes.length === 0) return policy?.fallbackMethodScopes ?? scopes; + const authoritative = policy?.authoritativeScopes; + return authoritative ? scopes.filter((scope) => authoritative[scope] !== undefined) : scopes; + }, + userConsent: isGoogleUserConsentOAuthScope, + covers: (consent, scope) => { + if (consent === scope) return true; + if (!isGoogleUserConsentOAuthScope(scope)) return false; + const compacted = compactGoogleOAuthScopes([consent, scope]); + return compacted.length === 1 && compacted[0] === consent; + }, +}; // The converter returns executor-flavored Effects; run them with executor's // own effect instance so the runtime identities match. @@ -125,7 +149,7 @@ async function main(): Promise { googleOperationScopes( realGooglePaths(converted, discoveryDocuments, preset.id), discoveryDocuments, - scopeCovers, + scopeRules, preset.id, ), preset.id, @@ -136,6 +160,14 @@ async function main(): Promise { console.error(`${preset.id}: 0 operations after policy filtering, skipping`); continue; } + const unlisted = unlistedDiscoveryScopes( + hosted.document, + discoveryDocuments, + scopeRules, + preset.id, + ); + if (unlisted.length > 0) + throw new Error(`${preset.id}: operations omit scopes Google accepts:\n${unlisted.join("\n")}`); // The bundle converter titles everything "Google"; a hosted standalone // document should carry the service's own identity, since clients name an diff --git a/scripts/google-specs.ts b/scripts/google-specs.ts index b0e781ce..0f6124fd 100644 --- a/scripts/google-specs.ts +++ b/scripts/google-specs.ts @@ -32,8 +32,8 @@ * resource-name parameter, such as `name: "spaces/AAA/messages/BBB"`. * Operations that share one are told apart by their parameters' Discovery * patterns. `googleOperationScopes` lists every scope Discovery accepts for - * each operation that its converted scopes cover. Both need the Discovery - * documents, so only generation runs them. + * each operation that Google grants through user consent. Both need the + * Discovery documents, so only generation runs them. * `hostedGoogleSpec` rewrites the rest and refuses an operation whose path is * not real. Generation and the committed-spec test both run `hostedGoogleSpec`, * so a regenerated spec cannot reintroduce any of these mistakes. @@ -355,9 +355,17 @@ export function realGooglePaths( const OAUTH_SCHEME = "googleOAuth2"; const GOOGLE_SCOPES = "x-google-scopes"; -/** Whether a token granted `consent` may call a method that accepts `scope`. - * Generation passes executor's rule, the one its converter applied. */ -export type ScopeCovers = (consent: string, scope: string) => boolean; +/** Executor's Google scope rules. Generation passes the ones executor's + * converter applies, from the executor checkout it runs. */ +export interface GoogleScopeRules { + /** The scopes executor's policy for a Discovery service lets one of its + * methods accept, given the scopes Discovery lists for the method. */ + readonly methodScopes: (service: string, version: string, scopes: readonly string[]) => readonly string[]; + /** Whether Google grants a scope through an ordinary user's OAuth consent. */ + readonly userConsent: (scope: string) => boolean; + /** Whether a token granted `consent` may call a method that accepts `scope`. */ + readonly covers: (consent: string, scope: string) => boolean; +} /** The scopes an operation's security requirements name. */ function securityScopes(security: Json | undefined, at: string): readonly string[] { @@ -386,6 +394,87 @@ function discoveryScopeDescriptions(discovery: readonly JsonObject[]): ReadonlyM return descriptions; } +type ServiceMethod = { + readonly method: JsonObject; + readonly service: string; + readonly version: string; +}; + +/** Each Discovery method by id, with the service and version of the document + * that lists it, which is how executor's service policies are keyed. */ +function serviceMethods( + discovery: readonly JsonObject[], + at: string, +): ReadonlyMap { + return new Map( + discovery.flatMap((document, index) => { + const { name: service, version } = document; + if (typeof service !== "string" || typeof version !== "string") + throw new Error(`${at} Discovery document ${index}: no name or version`); + return [...discoveryMethods([document], at)].map( + ([id, method]) => [id, { method, service, version }] as const, + ); + }), + ); +} + +/** + * Orders scopes so that each comes after every other listed scope it covers: + * a broad scope follows the narrower ones it grants. Scopes neither of which + * covers the other are in name order. That is not a privilege ranking, since + * the coverage rule does not relate every pair (`gmail.compose` and + * `gmail.send` are unrelated under it, and sort by name). + * + * @throws when two scopes cover each other. + */ +function coverageOrder( + scopes: readonly string[], + covers: GoogleScopeRules["covers"], + at: string, +): readonly string[] { + const remaining = [...new Set(scopes)].sort(byText); + const ordered: string[] = []; + while (remaining.length > 0) { + const next = remaining.findIndex( + (scope) => !remaining.some((other) => other !== scope && covers(scope, other)), + ); + if (next < 0) throw new Error(`${at}: the scopes ${remaining.join(", ")} cover each other`); + ordered.push(...remaining.splice(next, 1)); + } + return ordered; +} + +/** The Discovery method an operation was converted from, by its operation id. + * The converter names a media upload `${id}Media`. */ +function sourceMethod( + methods: ReadonlyMap, + operation: JsonObject, +): ServiceMethod | undefined { + const id = operation.operationId; + if (typeof id !== "string") return undefined; + return ( + methods.get(id) ?? + (id.endsWith("Media") ? methods.get(id.slice(0, -"Media".length)) : undefined) + ); +} + +/** The scopes Discovery lists for a method that executor's service policy + * admits and that Google grants through user consent, in Discovery's order. */ +function acceptedScopes( + source: ServiceMethod, + rules: GoogleScopeRules, + at: string, +): readonly string[] { + const listed = + source.method.scopes === undefined + ? [] + : arrayAt(source.method.scopes, `${at} Discovery scopes`).map((scope) => { + if (typeof scope !== "string") throw new Error(`${at}: Discovery scopes are not names`); + return scope; + }); + return rules.methodScopes(source.service, source.version, listed).filter(rules.userConsent); +} + /** * Lists on each operation every scope Google accepts for it, not only the * broad scope the integration asks consent for. @@ -395,22 +484,30 @@ function discoveryScopeDescriptions(discovery: readonly JsonObject[]): ReadonlyM * `https://mail.google.com/`. Discovery lists the scopes each method accepts * (`users.messages.list` accepts `gmail.readonly`, `gmail.metadata`, * `gmail.modify` and full access). Each operation now names, as alternative - * security requirements, every scope Discovery lists for its method that one - * of its converted scopes covers, narrower scopes first, then the converted - * scopes. The covered rule keeps scopes Google will not grant through user - * consent out. An operation with no Discovery method or no Discovery scopes - * (executor's own Photos upload, Photos Picker's unscoped methods) keeps the - * scopes the converter gave it. The OAuth scheme declares every scope an - * operation names, with Discovery's description. Run after `realGooglePaths`. - * The input is not modified. + * security requirements: + * + * - every scope Discovery lists for its method, as executor's policy for the + * service admits it (Photos keeps only the scopes Google still grants), that + * Google grants through user consent; and + * - the scopes the converter gave it, so the integration's consent still + * reaches every operation. + * + * Whether a consent scope covers a Discovery scope does not matter here: the + * coverage table knows only some pairs, and `chat.messages.create` is accepted + * by `spaces.messages.create` although `chat.messages` is not known to cover + * it. The order is `coverageOrder`'s. An operation with no Discovery method or + * no security (executor's own Photos upload) keeps the scopes the converter + * gave it. The OAuth scheme declares every scope an operation names, with + * Discovery's description. Run after `realGooglePaths`. The input is not + * modified. */ export function googleOperationScopes( spec: JsonObject, discovery: readonly JsonObject[], - covers: ScopeCovers, + rules: GoogleScopeRules, at: string, ): JsonObject { - const methods = discoveryMethods(discovery, at); + const methods = serviceMethods(discovery, at); const descriptions = discoveryScopeDescriptions(discovery); const used = new Set(); const paths = Object.fromEntries( @@ -425,25 +522,10 @@ export function googleOperationScopes( const operation = objectAt(field, where); const granted = securityScopes(operation.security, `${where} security`); granted.forEach((scope) => used.add(scope)); - const id = operation.operationId; - const discoveryMethod = - typeof id !== "string" - ? undefined - : (methods.get(id) ?? - (id.endsWith("Media") ? methods.get(id.slice(0, -"Media".length)) : undefined)); - const accepted = discoveryMethod?.scopes; - if (accepted === undefined || granted.length === 0) return [method, operation]; - const narrower = arrayAt(accepted, `${where} Discovery scopes`) - .flatMap((scope) => { - if (typeof scope !== "string") - throw new Error(`${where}: Discovery scopes are not names`); - return granted.includes(scope) || - !granted.some((consent) => covers(consent, scope)) - ? [] - : [scope]; - }) - .sort(byText); - const scopes = [...narrower, ...granted]; + const source = sourceMethod(methods, operation); + if (source === undefined || granted.length === 0) return [method, operation]; + const accepted = acceptedScopes(source, rules, where); + const scopes = [...coverageOrder([...accepted, ...granted], rules.covers, where)]; scopes.forEach((scope) => used.add(scope)); return [ method, @@ -489,6 +571,34 @@ export function googleOperationScopes( }; } +/** + * The scopes Google accepts for an operation's Discovery method through user + * consent (`acceptedScopes`) that the operation does not name, one + * ` : ` line each. Generation fails on any, so no later + * step can drop a scope from a published spec. + */ +export function unlistedDiscoveryScopes( + spec: JsonObject, + discovery: readonly JsonObject[], + rules: GoogleScopeRules, + at: string, +): readonly string[] { + const methods = serviceMethods(discovery, at); + return Object.entries(objectAt(spec.paths, `${at} paths`)).flatMap(([path, value]) => + itemOperations(objectAt(value, `${at} ${path}`), `${at} ${path}`).flatMap( + ({ method, operation }) => { + const where = `${at} ${method} ${path}`; + const source = sourceMethod(methods, operation); + if (source === undefined) return []; + const named = securityScopes(operation.security, `${where} security`); + return acceptedScopes(source, rules, where) + .filter((scope) => !named.includes(scope)) + .map((scope) => `${method} ${path}: ${scope}`); + }, + ), + ); +} + // --------------------------------------------------------------------------- // Hosted form // --------------------------------------------------------------------------- From dc6d390a1ca1940be1c1e18b1bc4b1359ac454ea Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Wed, 7 Oct 2026 14:45:38 -0700 Subject: [PATCH 3/4] List Google scope alternatives in x-google-scopes, not in security or the flow Importers request every scope an OAuth flow declares, and legacy Executor records each operation's security as its required scopes, so listing the narrower alternatives there widened the consent a Docs or BigQuery import asks for. Security and the flow now stay as executor's converter writes them. --- scripts/generate-google-specs.ts | 42 ++----- scripts/google-specs.ts | 200 +++++++++++++++++++++++-------- 2 files changed, 157 insertions(+), 85 deletions(-) diff --git a/scripts/generate-google-specs.ts b/scripts/generate-google-specs.ts index 0b25a4af..d8bb79b4 100644 --- a/scripts/generate-google-specs.ts +++ b/scripts/generate-google-specs.ts @@ -10,9 +10,10 @@ * corrects what that converter's multi-service bundle output gets wrong for a * document describing one service (scripts/google-specs.ts): realGooglePaths * keys every operation by its real path, googleOperationScopes lists every - * scope Google accepts for each operation, and hostedGoogleSpec fixes the - * document server, types Discovery's string-encoded defaults, writes everything - * in a fixed order and refuses any operation left on a placeholder path. + * scope Google accepts for each operation in its x-google-scopes, and + * hostedGoogleSpec fixes the document server, types Discovery's string-encoded + * defaults, writes everything in a fixed order and refuses any operation left + * on a placeholder path. * * The converter is not part of the published @executor-js/plugin-openapi * package, so it is imported from a local executor (v1) checkout, along with @@ -29,13 +30,13 @@ import { existsSync, mkdirSync, writeFileSync } from "node:fs"; import { join, resolve, dirname } from "node:path"; import { fileURLToPath } from "node:url"; import { + executorScopeRules, googleOperationScopes, hostedGoogleSpec, isJsonObject, parseJsonObject, realGooglePaths, unlistedDiscoveryScopes, - type GoogleScopeRules, } from "./google-specs.ts"; const ROOT = join(dirname(fileURLToPath(import.meta.url)), ".."); @@ -59,34 +60,11 @@ const { compactGoogleOAuthScopes, isGoogleUserConsentOAuthScope } = await import join(providerDir, "oauth-scopes.ts") ); const { googleDiscoveryPolicyFor } = await import(join(providerDir, "service-policy.ts")); - -/** Executor's Google scope rules, as its converter applies them. Neither - * discoveryMethodScopesForService nor googleScopeCovers in discovery.ts is - * exported, so these repeat them over the exported parts. One difference: - * compaction renames `userinfo.email` to `email` rather than dropping it, so - * googleScopeCovers has every scope cover the identity scopes, and each - * identity scope cover the other. Here a scope covers another only when - * compaction drops the other and keeps it unchanged. */ -const scopeRules: GoogleScopeRules = { - methodScopes: (service, version, scopes) => { - const policy = googleDiscoveryPolicyFor(service, version) as - | { - readonly authoritativeScopes?: Readonly>; - readonly fallbackMethodScopes?: readonly string[]; - } - | undefined; - if (scopes.length === 0) return policy?.fallbackMethodScopes ?? scopes; - const authoritative = policy?.authoritativeScopes; - return authoritative ? scopes.filter((scope) => authoritative[scope] !== undefined) : scopes; - }, - userConsent: isGoogleUserConsentOAuthScope, - covers: (consent, scope) => { - if (consent === scope) return true; - if (!isGoogleUserConsentOAuthScope(scope)) return false; - const compacted = compactGoogleOAuthScopes([consent, scope]); - return compacted.length === 1 && compacted[0] === consent; - }, -}; +const scopeRules = executorScopeRules({ + googleDiscoveryPolicyFor, + isGoogleUserConsentOAuthScope, + compactGoogleOAuthScopes, +}); // The converter returns executor-flavored Effects; run them with executor's // own effect instance so the runtime identities match. diff --git a/scripts/google-specs.ts b/scripts/google-specs.ts index 0f6124fd..62143ada 100644 --- a/scripts/google-specs.ts +++ b/scripts/google-specs.ts @@ -22,7 +22,7 @@ * - It copies Discovery defaults verbatim, and Discovery encodes every default * as a string, so a boolean parameter declares `default: "true"`. * - It gives each operation only the consent scopes that cover its method, so - * every Gmail operation, reads included, declares full mail access + * every Gmail operation, reads included, names only full mail access * (`https://mail.google.com/`) although Google accepts `gmail.readonly`. * - It writes paths, schemas, properties and parameters in Discovery's order, * which changes on every fetch. @@ -31,9 +31,11 @@ * method and template no other operation shares keeps that template and its * resource-name parameter, such as `name: "spaces/AAA/messages/BBB"`. * Operations that share one are told apart by their parameters' Discovery - * patterns. `googleOperationScopes` lists every scope Discovery accepts for - * each operation that Google grants through user consent. Both need the - * Discovery documents, so only generation runs them. + * patterns. `googleOperationScopes` lists in each operation's + * `x-google-scopes` every scope Discovery accepts for it that Google grants + * through user consent, and leaves `security` and the OAuth flow as the + * converter wrote them. Both need the Discovery documents, so only generation + * runs them. * `hostedGoogleSpec` rewrites the rest and refuses an operation whose path is * not real. Generation and the committed-spec test both run `hostedGoogleSpec`, * so a regenerated spec cannot reintroduce any of these mistakes. @@ -355,8 +357,8 @@ export function realGooglePaths( const OAUTH_SCHEME = "googleOAuth2"; const GOOGLE_SCOPES = "x-google-scopes"; -/** Executor's Google scope rules. Generation passes the ones executor's - * converter applies, from the executor checkout it runs. */ +/** Executor's Google scope rules. Generation builds them with + * `executorScopeRules` from the executor checkout it runs. */ export interface GoogleScopeRules { /** The scopes executor's policy for a Discovery service lets one of its * methods accept, given the scopes Discovery lists for the method. */ @@ -367,6 +369,51 @@ export interface GoogleScopeRules { readonly covers: (consent: string, scope: string) => boolean; } +/** The parts of executor's Google provider (`service-policy.ts`, + * `oauth-scopes.ts`) that `executorScopeRules` is built from. */ +export interface ExecutorGoogleScopes { + readonly googleDiscoveryPolicyFor: ( + service: string, + version: string, + ) => + | { + readonly authoritativeScopes?: Readonly>; + readonly fallbackMethodScopes?: readonly string[]; + } + | undefined; + readonly isGoogleUserConsentOAuthScope: (scope: string) => boolean; + readonly compactGoogleOAuthScopes: (scopes: Iterable) => string[]; +} + +/** + * Executor's Google scope rules, as its converter applies them. Neither + * `discoveryMethodScopesForService` nor `googleScopeCovers` in executor's + * `discovery.ts` is exported, so these repeat them over the exported parts. + * + * One difference: compaction renames the identity scopes (`userinfo.email` + * becomes `email`) rather than dropping them, so `googleScopeCovers` has every + * scope cover them. Here a scope covers another only when compacting the two + * keeps the first unchanged and drops the second. + */ +export function executorScopeRules(executor: ExecutorGoogleScopes): GoogleScopeRules { + const { compactGoogleOAuthScopes, isGoogleUserConsentOAuthScope } = executor; + return { + methodScopes: (service, version, scopes) => { + const policy = executor.googleDiscoveryPolicyFor(service, version); + if (scopes.length === 0) return policy?.fallbackMethodScopes ?? scopes; + const authoritative = policy?.authoritativeScopes; + return authoritative ? scopes.filter((scope) => authoritative[scope] !== undefined) : scopes; + }, + userConsent: isGoogleUserConsentOAuthScope, + covers: (consent, scope) => { + if (consent === scope) return true; + if (!isGoogleUserConsentOAuthScope(scope)) return false; + const compacted = compactGoogleOAuthScopes([consent, scope]); + return compacted.length === 1 && compacted[0] === consent; + }, + }; +} + /** The scopes an operation's security requirements name. */ function securityScopes(security: Json | undefined, at: string): readonly string[] { if (security === undefined) return []; @@ -420,10 +467,13 @@ function serviceMethods( /** * Orders scopes so that each comes after every other listed scope it covers: - * a broad scope follows the narrower ones it grants. Scopes neither of which - * covers the other are in name order. That is not a privilege ranking, since + * a broad scope follows the narrower ones it grants. It is a topological sort + * that, at each step, takes the first scope by name among those that cover no + * scope still waiting. So unrelated scopes are not always in name order: + * `drive` waits for `drive.file`, which it covers, and `drive.apps.readonly`, + * which it does not, can come before it. Nor is it a privilege ranking, since * the coverage rule does not relate every pair (`gmail.compose` and - * `gmail.send` are unrelated under it, and sort by name). + * `gmail.send` are unrelated under it). * * @throws when two scopes cover each other. */ @@ -475,31 +525,49 @@ function acceptedScopes( return rules.methodScopes(source.service, source.version, listed).filter(rules.userConsent); } +/** The scope names in an `x-google-scopes` list. */ +function scopeList(value: Json | undefined, at: string): readonly string[] { + if (value === undefined) return []; + return arrayAt(value, at).map((scope) => { + if (typeof scope !== "string") throw new Error(`${at}: expected scope names`); + return scope; + }); +} + /** - * Lists on each operation every scope Google accepts for it, not only the - * broad scope the integration asks consent for. + * Lists in each operation's `x-google-scopes` every scope Google accepts for + * it, not only the broad scope the integration asks consent for. * * Executor's converter gives each operation the consent scopes that cover its - * method, so every Gmail operation, reads included, declared only + * method, so every Gmail operation, reads included, named only * `https://mail.google.com/`. Discovery lists the scopes each method accepts * (`users.messages.list` accepts `gmail.readonly`, `gmail.metadata`, - * `gmail.modify` and full access). Each operation now names, as alternative - * security requirements: + * `gmail.modify` and full access). Each operation's `x-google-scopes` now + * names, in `coverageOrder`: * * - every scope Discovery lists for its method, as executor's policy for the * service admits it (Photos keeps only the scopes Google still grants), that * Google grants through user consent; and - * - the scopes the converter gave it, so the integration's consent still - * reaches every operation. + * - the scopes its security requirements name. * * Whether a consent scope covers a Discovery scope does not matter here: the * coverage table knows only some pairs, and `chat.messages.create` is accepted * by `spaces.messages.create` although `chat.messages` is not known to cover - * it. The order is `coverageOrder`'s. An operation with no Discovery method or - * no security (executor's own Photos upload) keeps the scopes the converter - * gave it. The OAuth scheme declares every scope an operation names, with - * Discovery's description. Run after `realGooglePaths`. The input is not - * modified. + * it. The OAuth scheme's own `x-google-scopes` maps every scope an operation + * lists to its description, the flow's or else Discovery's, in name order. An + * operation with no Discovery method or no security (executor's own Photos + * upload) keeps the list the converter gave it. + * + * The operations' `security` and the flow's `scopes` stay exactly as the + * converter wrote them. Importers take the consent they request from those: + * legacy Executor's URL importer (`@executor-js/plugin-openapi`) asks for + * every scope the flow declares and records each operation's `security` as + * its required scopes. Listing the alternatives there would make a Docs + * connection ask for all of Drive, and OpenAPI linters reject a requirement + * naming a scope its flow does not declare. Extensions are valid on Operation + * and Security Scheme Objects, and those importers ignore them. + * + * Run after `realGooglePaths`. The input is not modified. */ export function googleOperationScopes( spec: JsonObject, @@ -521,20 +589,17 @@ export function googleOperationScopes( const where = `${at} ${method} ${path}`; const operation = objectAt(field, where); const granted = securityScopes(operation.security, `${where} security`); - granted.forEach((scope) => used.add(scope)); const source = sourceMethod(methods, operation); - if (source === undefined || granted.length === 0) return [method, operation]; + if (source === undefined || granted.length === 0) { + scopeList(operation[GOOGLE_SCOPES], `${where} ${GOOGLE_SCOPES}`).forEach((scope) => + used.add(scope), + ); + return [method, operation]; + } const accepted = acceptedScopes(source, rules, where); const scopes = [...coverageOrder([...accepted, ...granted], rules.covers, where)]; scopes.forEach((scope) => used.add(scope)); - return [ - method, - { - ...operation, - security: scopes.map((scope) => ({ [OAUTH_SCHEME]: [scope] })), - [GOOGLE_SCOPES]: scopes, - }, - ]; + return [method, { ...operation, [GOOGLE_SCOPES]: scopes }]; }), ), ]; @@ -547,33 +612,30 @@ export function googleOperationScopes( const flows = objectAt(scheme.flows, `${at} ${OAUTH_SCHEME} flows`); const flow = objectAt(flows.authorizationCode, `${at} ${OAUTH_SCHEME} authorizationCode`); const declared = objectAt(flow.scopes, `${at} ${OAUTH_SCHEME} scopes`); - const scopes: JsonObject = { ...declared }; - for (const scope of used) { - const description = declared[scope]; - scopes[scope] = - typeof description === "string" && description !== "" - ? description - : (descriptions.get(scope) ?? ""); - } + const described = Object.fromEntries( + [...used].sort(byText).map((scope) => { + const description = declared[scope]; + return [ + scope, + typeof description === "string" && description !== "" + ? description + : (descriptions.get(scope) ?? ""), + ]; + }), + ); return { ...spec, paths, components: { ...components, - securitySchemes: { - ...schemes, - [OAUTH_SCHEME]: { - ...scheme, - flows: { ...flows, authorizationCode: { ...flow, scopes } }, - }, - }, + securitySchemes: { ...schemes, [OAUTH_SCHEME]: { ...scheme, [GOOGLE_SCOPES]: described } }, }, }; } /** * The scopes Google accepts for an operation's Discovery method through user - * consent (`acceptedScopes`) that the operation does not name, one + * consent (`acceptedScopes`) that its `x-google-scopes` does not list, one * ` : ` line each. Generation fails on any, so no later * step can drop a scope from a published spec. */ @@ -590,9 +652,9 @@ export function unlistedDiscoveryScopes( const where = `${at} ${method} ${path}`; const source = sourceMethod(methods, operation); if (source === undefined) return []; - const named = securityScopes(operation.security, `${where} security`); + const listed = scopeList(operation[GOOGLE_SCOPES], `${where} ${GOOGLE_SCOPES}`); return acceptedScopes(source, rules, where) - .filter((scope) => !named.includes(scope)) + .filter((scope) => !listed.includes(scope)) .map((scope) => `${method} ${path}: ${scope}`); }, ), @@ -626,14 +688,17 @@ export interface HostedGoogleSpec { * order, so the same service always produces the same document. * * @throws when an operation is not keyed by a real path, when a security - * requirement names a scope its OAuth scheme does not declare, when the - * operations span more than one origin, or when a default cannot be read as its schema's - * type. Each is a change in Google's documents or executor's converter that a - * person must look at, not something to publish. + * requirement names a scope its OAuth scheme does not declare, when an + * operation's `x-google-scopes` leaves out a scope its security names or lists + * one the OAuth scheme does not describe, when the operations span more than + * one origin, or when a default cannot be read as its schema's type. Each is a + * change in Google's documents or executor's converter that a person must look + * at, not something to publish. */ export function hostedGoogleSpec(spec: JsonObject, at: string): HostedGoogleSpec | undefined { assertRealPaths(spec, at); assertDeclaredScopes(spec, at); + assertListedScopes(spec, at); const routed = withOperationServer(spec, at); if (routed === undefined) return undefined; const schemas = withSchemas(routed.document, at, (schema, where) => @@ -721,6 +786,35 @@ function assertDeclaredScopes(spec: JsonObject, at: string): void { check(operation.security, `${at} ${method} ${path}`); } +/** + * @throws when an operation's `x-google-scopes` leaves out a scope its security + * requirements name, or lists a scope that neither the OAuth flow declares nor + * the OAuth scheme's `x-google-scopes` describes. + */ +function assertListedScopes(spec: JsonObject, at: string): void { + const components = isJsonObject(spec.components) ? spec.components : {}; + const schemes = isJsonObject(components.securitySchemes) ? components.securitySchemes : {}; + const scheme = isJsonObject(schemes[OAUTH_SCHEME]) ? schemes[OAUTH_SCHEME] : {}; + const flows = isJsonObject(scheme.flows) ? scheme.flows : {}; + const flow = isJsonObject(flows.authorizationCode) ? flows.authorizationCode : {}; + const known = new Set([ + ...Object.keys(isJsonObject(flow.scopes) ? flow.scopes : {}), + ...Object.keys(scheme[GOOGLE_SCOPES] === undefined ? {} : objectAt(scheme[GOOGLE_SCOPES], `${at} ${OAUTH_SCHEME} ${GOOGLE_SCOPES}`)), + ]); + for (const [path, value] of Object.entries(objectAt(spec.paths, `${at} paths`))) + for (const { method, operation } of itemOperations(objectAt(value, `${at} ${path}`), `${at} ${path}`)) { + if (operation[GOOGLE_SCOPES] === undefined) continue; + const where = `${at} ${method} ${path}`; + const listed = scopeList(operation[GOOGLE_SCOPES], `${where} ${GOOGLE_SCOPES}`); + for (const scope of securityScopes(operation.security, `${where} security`)) + if (!listed.includes(scope)) + throw new Error(`${where}: ${GOOGLE_SCOPES} leaves out the security scope ${JSON.stringify(scope)}`); + for (const scope of listed) + if (!known.has(scope)) + throw new Error(`${where}: ${OAUTH_SCHEME} does not describe the listed scope ${JSON.stringify(scope)}`); + } +} + function singleServerUrl(servers: Json | undefined, at: string): string | undefined { if (servers === undefined) return undefined; if (!Array.isArray(servers) || servers.length !== 1) From 06a66d1d72b95186d9aa79c763ae1feed446bf48 Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Wed, 7 Oct 2026 14:55:17 -0700 Subject: [PATCH 4/4] Explain how to read x-google-scopes in the CLI skill --- cli/src/skill.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/cli/src/skill.md b/cli/src/skill.md index 5439f3ef..dda0cb5b 100644 --- a/cli/src/skill.md +++ b/cli/src/skill.md @@ -56,6 +56,16 @@ How to read the response: entry's `use[]` lists credentials needed **together (AND)**; `status: "unknown"` means not yet determined. +For a Google `http` surface, fetch its `spec` to choose narrower OAuth scopes. +Each operation's `x-google-scopes` lists the scopes Google accepts for it; +any **one** of them is enough. The order is not a privilege ranking, and +importers keep requesting the broad scopes in `security`, so pick the +narrowest scope yourself. The `x-google-scopes` map on the `googleOAuth2` +security scheme describes each scope. Example: to list and read Gmail +messages, request `https://www.googleapis.com/auth/gmail.readonly`, not +`https://mail.google.com/` (`gmail.metadata` also lists messages but can't +read their bodies). + Not found means the domain isn't cataloged yet — escalate to step 3. **3. Detect / discover** when the lookup came back empty or stale: