diff --git a/cli/src/skill.md b/cli/src/skill.md index 5439f3ef..dda0cb5b 100644 --- a/cli/src/skill.md +++ b/cli/src/skill.md @@ -56,6 +56,16 @@ How to read the response: entry's `use[]` lists credentials needed **together (AND)**; `status: "unknown"` means not yet determined. +For a Google `http` surface, fetch its `spec` to choose narrower OAuth scopes. +Each operation's `x-google-scopes` lists the scopes Google accepts for it; +any **one** of them is enough. The order is not a privilege ranking, and +importers keep requesting the broad scopes in `security`, so pick the +narrowest scope yourself. The `x-google-scopes` map on the `googleOAuth2` +security scheme describes each scope. Example: to list and read Gmail +messages, request `https://www.googleapis.com/auth/gmail.readonly`, not +`https://mail.google.com/` (`gmail.metadata` also lists messages but can't +read their bodies). + Not found means the domain isn't cataloged yet — escalate to step 3. **3. Detect / discover** when the lookup came back empty or stale: diff --git a/scripts/generate-google-specs.ts b/scripts/generate-google-specs.ts index 218c96d5..d8bb79b4 100644 --- a/scripts/generate-google-specs.ts +++ b/scripts/generate-google-specs.ts @@ -9,9 +9,11 @@ * converter — the exact pipeline the product executes at add time — and then * corrects what that converter's multi-service bundle output gets wrong for a * document describing one service (scripts/google-specs.ts): realGooglePaths - * keys every operation by its real path, and hostedGoogleSpec fixes the - * document server, types Discovery's string-encoded defaults, writes everything - * in a fixed order and refuses any operation left on a placeholder path. + * keys every operation by its real path, googleOperationScopes lists every + * scope Google accepts for each operation in its x-google-scopes, and + * hostedGoogleSpec fixes the document server, types Discovery's string-encoded + * defaults, writes everything in a fixed order and refuses any operation left + * on a placeholder path. * * The converter is not part of the published @executor-js/plugin-openapi * package, so it is imported from a local executor (v1) checkout, along with @@ -28,10 +30,13 @@ import { existsSync, mkdirSync, writeFileSync } from "node:fs"; import { join, resolve, dirname } from "node:path"; import { fileURLToPath } from "node:url"; import { + executorScopeRules, + googleOperationScopes, hostedGoogleSpec, isJsonObject, parseJsonObject, realGooglePaths, + unlistedDiscoveryScopes, } from "./google-specs.ts"; const ROOT = join(dirname(fileURLToPath(import.meta.url)), ".."); @@ -51,6 +56,16 @@ const { googleOpenApiPresets, googleCatalogOAuthScopesForPreset } = await import const { convertGoogleDiscoveryBundleToOpenApi, normalizeGoogleDiscoveryUrl } = await import( join(providerDir, "discovery.ts") ); +const { compactGoogleOAuthScopes, isGoogleUserConsentOAuthScope } = await import( + join(providerDir, "oauth-scopes.ts") +); +const { googleDiscoveryPolicyFor } = await import(join(providerDir, "service-policy.ts")); +const scopeRules = executorScopeRules({ + googleDiscoveryPolicyFor, + isGoogleUserConsentOAuthScope, + compactGoogleOAuthScopes, +}); + // The converter returns executor-flavored Effects; run them with executor's // own effect instance so the runtime identities match. const { Effect } = await import(join(EXECUTOR, "node_modules/effect/dist/index.js")); @@ -109,7 +124,12 @@ async function main(): Promise { parseJsonObject(documentText, `${preset.id} Discovery document ${index}`), ); const hosted = hostedGoogleSpec( - realGooglePaths(converted, discoveryDocuments, preset.id), + googleOperationScopes( + realGooglePaths(converted, discoveryDocuments, preset.id), + discoveryDocuments, + scopeRules, + preset.id, + ), preset.id, ); if (hosted === undefined) { @@ -118,6 +138,14 @@ async function main(): Promise { console.error(`${preset.id}: 0 operations after policy filtering, skipping`); continue; } + const unlisted = unlistedDiscoveryScopes( + hosted.document, + discoveryDocuments, + scopeRules, + preset.id, + ); + if (unlisted.length > 0) + throw new Error(`${preset.id}: operations omit scopes Google accepts:\n${unlisted.join("\n")}`); // The bundle converter titles everything "Google"; a hosted standalone // document should carry the service's own identity, since clients name an diff --git a/scripts/google-specs.ts b/scripts/google-specs.ts index 129c6425..62143ada 100644 --- a/scripts/google-specs.ts +++ b/scripts/google-specs.ts @@ -21,6 +21,9 @@ * uses. * - It copies Discovery defaults verbatim, and Discovery encodes every default * as a string, so a boolean parameter declares `default: "true"`. + * - It gives each operation only the consent scopes that cover its method, so + * every Gmail operation, reads included, names only full mail access + * (`https://mail.google.com/`) although Google accepts `gmail.readonly`. * - It writes paths, schemas, properties and parameters in Discovery's order, * which changes on every fetch. * @@ -28,7 +31,11 @@ * method and template no other operation shares keeps that template and its * resource-name parameter, such as `name: "spaces/AAA/messages/BBB"`. * Operations that share one are told apart by their parameters' Discovery - * patterns. It needs the Discovery documents, so only generation runs it. + * patterns. `googleOperationScopes` lists in each operation's + * `x-google-scopes` every scope Discovery accepts for it that Google grants + * through user consent, and leaves `security` and the OAuth flow as the + * converter wrote them. Both need the Discovery documents, so only generation + * runs them. * `hostedGoogleSpec` rewrites the rest and refuses an operation whose path is * not real. Generation and the committed-spec test both run `hostedGoogleSpec`, * so a regenerated spec cannot reintroduce any of these mistakes. @@ -343,6 +350,317 @@ export function realGooglePaths( return { ...spec, paths }; } +// --------------------------------------------------------------------------- +// Operation scopes +// --------------------------------------------------------------------------- + +const OAUTH_SCHEME = "googleOAuth2"; +const GOOGLE_SCOPES = "x-google-scopes"; + +/** Executor's Google scope rules. Generation builds them with + * `executorScopeRules` from the executor checkout it runs. */ +export interface GoogleScopeRules { + /** The scopes executor's policy for a Discovery service lets one of its + * methods accept, given the scopes Discovery lists for the method. */ + readonly methodScopes: (service: string, version: string, scopes: readonly string[]) => readonly string[]; + /** Whether Google grants a scope through an ordinary user's OAuth consent. */ + readonly userConsent: (scope: string) => boolean; + /** Whether a token granted `consent` may call a method that accepts `scope`. */ + readonly covers: (consent: string, scope: string) => boolean; +} + +/** The parts of executor's Google provider (`service-policy.ts`, + * `oauth-scopes.ts`) that `executorScopeRules` is built from. */ +export interface ExecutorGoogleScopes { + readonly googleDiscoveryPolicyFor: ( + service: string, + version: string, + ) => + | { + readonly authoritativeScopes?: Readonly>; + readonly fallbackMethodScopes?: readonly string[]; + } + | undefined; + readonly isGoogleUserConsentOAuthScope: (scope: string) => boolean; + readonly compactGoogleOAuthScopes: (scopes: Iterable) => string[]; +} + +/** + * Executor's Google scope rules, as its converter applies them. Neither + * `discoveryMethodScopesForService` nor `googleScopeCovers` in executor's + * `discovery.ts` is exported, so these repeat them over the exported parts. + * + * One difference: compaction renames the identity scopes (`userinfo.email` + * becomes `email`) rather than dropping them, so `googleScopeCovers` has every + * scope cover them. Here a scope covers another only when compacting the two + * keeps the first unchanged and drops the second. + */ +export function executorScopeRules(executor: ExecutorGoogleScopes): GoogleScopeRules { + const { compactGoogleOAuthScopes, isGoogleUserConsentOAuthScope } = executor; + return { + methodScopes: (service, version, scopes) => { + const policy = executor.googleDiscoveryPolicyFor(service, version); + if (scopes.length === 0) return policy?.fallbackMethodScopes ?? scopes; + const authoritative = policy?.authoritativeScopes; + return authoritative ? scopes.filter((scope) => authoritative[scope] !== undefined) : scopes; + }, + userConsent: isGoogleUserConsentOAuthScope, + covers: (consent, scope) => { + if (consent === scope) return true; + if (!isGoogleUserConsentOAuthScope(scope)) return false; + const compacted = compactGoogleOAuthScopes([consent, scope]); + return compacted.length === 1 && compacted[0] === consent; + }, + }; +} + +/** The scopes an operation's security requirements name. */ +function securityScopes(security: Json | undefined, at: string): readonly string[] { + if (security === undefined) return []; + return arrayAt(security, at).flatMap((requirement) => + Object.values(objectAt(requirement, at)).flatMap((scopes) => + arrayAt(scopes, at).map((scope) => { + if (typeof scope !== "string") throw new Error(`${at}: expected scope names`); + return scope; + }), + ), + ); +} + +/** The scope descriptions the Discovery documents declare. */ +function discoveryScopeDescriptions(discovery: readonly JsonObject[]): ReadonlyMap { + const descriptions = new Map(); + for (const document of discovery) { + const auth = isJsonObject(document.auth) ? document.auth.oauth2 : undefined; + const scopes = isJsonObject(auth) ? auth.scopes : undefined; + if (isJsonObject(scopes)) + for (const [scope, value] of Object.entries(scopes)) + if (isJsonObject(value) && typeof value.description === "string") + descriptions.set(scope, value.description); + } + return descriptions; +} + +type ServiceMethod = { + readonly method: JsonObject; + readonly service: string; + readonly version: string; +}; + +/** Each Discovery method by id, with the service and version of the document + * that lists it, which is how executor's service policies are keyed. */ +function serviceMethods( + discovery: readonly JsonObject[], + at: string, +): ReadonlyMap { + return new Map( + discovery.flatMap((document, index) => { + const { name: service, version } = document; + if (typeof service !== "string" || typeof version !== "string") + throw new Error(`${at} Discovery document ${index}: no name or version`); + return [...discoveryMethods([document], at)].map( + ([id, method]) => [id, { method, service, version }] as const, + ); + }), + ); +} + +/** + * Orders scopes so that each comes after every other listed scope it covers: + * a broad scope follows the narrower ones it grants. It is a topological sort + * that, at each step, takes the first scope by name among those that cover no + * scope still waiting. So unrelated scopes are not always in name order: + * `drive` waits for `drive.file`, which it covers, and `drive.apps.readonly`, + * which it does not, can come before it. Nor is it a privilege ranking, since + * the coverage rule does not relate every pair (`gmail.compose` and + * `gmail.send` are unrelated under it). + * + * @throws when two scopes cover each other. + */ +function coverageOrder( + scopes: readonly string[], + covers: GoogleScopeRules["covers"], + at: string, +): readonly string[] { + const remaining = [...new Set(scopes)].sort(byText); + const ordered: string[] = []; + while (remaining.length > 0) { + const next = remaining.findIndex( + (scope) => !remaining.some((other) => other !== scope && covers(scope, other)), + ); + if (next < 0) throw new Error(`${at}: the scopes ${remaining.join(", ")} cover each other`); + ordered.push(...remaining.splice(next, 1)); + } + return ordered; +} + +/** The Discovery method an operation was converted from, by its operation id. + * The converter names a media upload `${id}Media`. */ +function sourceMethod( + methods: ReadonlyMap, + operation: JsonObject, +): ServiceMethod | undefined { + const id = operation.operationId; + if (typeof id !== "string") return undefined; + return ( + methods.get(id) ?? + (id.endsWith("Media") ? methods.get(id.slice(0, -"Media".length)) : undefined) + ); +} + +/** The scopes Discovery lists for a method that executor's service policy + * admits and that Google grants through user consent, in Discovery's order. */ +function acceptedScopes( + source: ServiceMethod, + rules: GoogleScopeRules, + at: string, +): readonly string[] { + const listed = + source.method.scopes === undefined + ? [] + : arrayAt(source.method.scopes, `${at} Discovery scopes`).map((scope) => { + if (typeof scope !== "string") throw new Error(`${at}: Discovery scopes are not names`); + return scope; + }); + return rules.methodScopes(source.service, source.version, listed).filter(rules.userConsent); +} + +/** The scope names in an `x-google-scopes` list. */ +function scopeList(value: Json | undefined, at: string): readonly string[] { + if (value === undefined) return []; + return arrayAt(value, at).map((scope) => { + if (typeof scope !== "string") throw new Error(`${at}: expected scope names`); + return scope; + }); +} + +/** + * Lists in each operation's `x-google-scopes` every scope Google accepts for + * it, not only the broad scope the integration asks consent for. + * + * Executor's converter gives each operation the consent scopes that cover its + * method, so every Gmail operation, reads included, named only + * `https://mail.google.com/`. Discovery lists the scopes each method accepts + * (`users.messages.list` accepts `gmail.readonly`, `gmail.metadata`, + * `gmail.modify` and full access). Each operation's `x-google-scopes` now + * names, in `coverageOrder`: + * + * - every scope Discovery lists for its method, as executor's policy for the + * service admits it (Photos keeps only the scopes Google still grants), that + * Google grants through user consent; and + * - the scopes its security requirements name. + * + * Whether a consent scope covers a Discovery scope does not matter here: the + * coverage table knows only some pairs, and `chat.messages.create` is accepted + * by `spaces.messages.create` although `chat.messages` is not known to cover + * it. The OAuth scheme's own `x-google-scopes` maps every scope an operation + * lists to its description, the flow's or else Discovery's, in name order. An + * operation with no Discovery method or no security (executor's own Photos + * upload) keeps the list the converter gave it. + * + * The operations' `security` and the flow's `scopes` stay exactly as the + * converter wrote them. Importers take the consent they request from those: + * legacy Executor's URL importer (`@executor-js/plugin-openapi`) asks for + * every scope the flow declares and records each operation's `security` as + * its required scopes. Listing the alternatives there would make a Docs + * connection ask for all of Drive, and OpenAPI linters reject a requirement + * naming a scope its flow does not declare. Extensions are valid on Operation + * and Security Scheme Objects, and those importers ignore them. + * + * Run after `realGooglePaths`. The input is not modified. + */ +export function googleOperationScopes( + spec: JsonObject, + discovery: readonly JsonObject[], + rules: GoogleScopeRules, + at: string, +): JsonObject { + const methods = serviceMethods(discovery, at); + const descriptions = discoveryScopeDescriptions(discovery); + const used = new Set(); + const paths = Object.fromEntries( + Object.entries(objectAt(spec.paths, `${at} paths`)).map(([path, value]) => { + const item = objectAt(value, `${at} ${path}`); + return [ + path, + Object.fromEntries( + Object.entries(item).map(([method, field]): [string, Json] => { + if (!HTTP_METHODS.has(method)) return [method, field]; + const where = `${at} ${method} ${path}`; + const operation = objectAt(field, where); + const granted = securityScopes(operation.security, `${where} security`); + const source = sourceMethod(methods, operation); + if (source === undefined || granted.length === 0) { + scopeList(operation[GOOGLE_SCOPES], `${where} ${GOOGLE_SCOPES}`).forEach((scope) => + used.add(scope), + ); + return [method, operation]; + } + const accepted = acceptedScopes(source, rules, where); + const scopes = [...coverageOrder([...accepted, ...granted], rules.covers, where)]; + scopes.forEach((scope) => used.add(scope)); + return [method, { ...operation, [GOOGLE_SCOPES]: scopes }]; + }), + ), + ]; + }), + ); + + const components = objectAt(spec.components, `${at} components`); + const schemes = objectAt(components.securitySchemes, `${at} securitySchemes`); + const scheme = objectAt(schemes[OAUTH_SCHEME], `${at} ${OAUTH_SCHEME}`); + const flows = objectAt(scheme.flows, `${at} ${OAUTH_SCHEME} flows`); + const flow = objectAt(flows.authorizationCode, `${at} ${OAUTH_SCHEME} authorizationCode`); + const declared = objectAt(flow.scopes, `${at} ${OAUTH_SCHEME} scopes`); + const described = Object.fromEntries( + [...used].sort(byText).map((scope) => { + const description = declared[scope]; + return [ + scope, + typeof description === "string" && description !== "" + ? description + : (descriptions.get(scope) ?? ""), + ]; + }), + ); + return { + ...spec, + paths, + components: { + ...components, + securitySchemes: { ...schemes, [OAUTH_SCHEME]: { ...scheme, [GOOGLE_SCOPES]: described } }, + }, + }; +} + +/** + * The scopes Google accepts for an operation's Discovery method through user + * consent (`acceptedScopes`) that its `x-google-scopes` does not list, one + * ` : ` line each. Generation fails on any, so no later + * step can drop a scope from a published spec. + */ +export function unlistedDiscoveryScopes( + spec: JsonObject, + discovery: readonly JsonObject[], + rules: GoogleScopeRules, + at: string, +): readonly string[] { + const methods = serviceMethods(discovery, at); + return Object.entries(objectAt(spec.paths, `${at} paths`)).flatMap(([path, value]) => + itemOperations(objectAt(value, `${at} ${path}`), `${at} ${path}`).flatMap( + ({ method, operation }) => { + const where = `${at} ${method} ${path}`; + const source = sourceMethod(methods, operation); + if (source === undefined) return []; + const listed = scopeList(operation[GOOGLE_SCOPES], `${where} ${GOOGLE_SCOPES}`); + return acceptedScopes(source, rules, where) + .filter((scope) => !listed.includes(scope)) + .map((scope) => `${method} ${path}: ${scope}`); + }, + ), + ); +} + // --------------------------------------------------------------------------- // Hosted form // --------------------------------------------------------------------------- @@ -369,13 +687,18 @@ export interface HostedGoogleSpec { * - Paths, methods, schemas, properties and parameters are written in a fixed * order, so the same service always produces the same document. * - * @throws when an operation is not keyed by a real path, when the operations - * span more than one origin, or when a default cannot be read as its schema's - * type. Each is a change in Google's documents or executor's converter that a - * person must look at, not something to publish. + * @throws when an operation is not keyed by a real path, when a security + * requirement names a scope its OAuth scheme does not declare, when an + * operation's `x-google-scopes` leaves out a scope its security names or lists + * one the OAuth scheme does not describe, when the operations span more than + * one origin, or when a default cannot be read as its schema's type. Each is a + * change in Google's documents or executor's converter that a person must look + * at, not something to publish. */ export function hostedGoogleSpec(spec: JsonObject, at: string): HostedGoogleSpec | undefined { assertRealPaths(spec, at); + assertDeclaredScopes(spec, at); + assertListedScopes(spec, at); const routed = withOperationServer(spec, at); if (routed === undefined) return undefined; const schemas = withSchemas(routed.document, at, (schema, where) => @@ -435,6 +758,63 @@ function assertRealPaths(spec: JsonObject, at: string): void { } } +/** @throws when a security requirement names a scope its OAuth scheme does not declare. */ +function assertDeclaredScopes(spec: JsonObject, at: string): void { + const components = isJsonObject(spec.components) ? spec.components : {}; + const schemes = isJsonObject(components.securitySchemes) ? components.securitySchemes : {}; + const declared = (name: string, where: string): JsonObject | undefined => { + const scheme = schemes[name]; + if (!isJsonObject(scheme)) throw new Error(`${where}: no security scheme ${name}`); + if (scheme.type !== "oauth2") return undefined; + const flows = objectAt(scheme.flows, `${at} ${name} flows`); + return objectAt(objectAt(flows.authorizationCode, `${at} ${name} authorizationCode`).scopes, `${at} ${name} scopes`); + }; + const check = (security: Json | undefined, where: string): void => { + if (security === undefined) return; + for (const requirement of arrayAt(security, `${where} security`)) + for (const [name, scopes] of Object.entries(objectAt(requirement, `${where} security`))) { + const known = declared(name, where); + if (known === undefined) continue; + for (const scope of arrayAt(scopes, `${where} security`)) + if (typeof scope !== "string" || !(scope in known)) + throw new Error(`${where}: ${name} does not declare the scope ${JSON.stringify(scope)}`); + } + }; + check(spec.security, at); + for (const [path, value] of Object.entries(objectAt(spec.paths, `${at} paths`))) + for (const { method, operation } of itemOperations(objectAt(value, `${at} ${path}`), `${at} ${path}`)) + check(operation.security, `${at} ${method} ${path}`); +} + +/** + * @throws when an operation's `x-google-scopes` leaves out a scope its security + * requirements name, or lists a scope that neither the OAuth flow declares nor + * the OAuth scheme's `x-google-scopes` describes. + */ +function assertListedScopes(spec: JsonObject, at: string): void { + const components = isJsonObject(spec.components) ? spec.components : {}; + const schemes = isJsonObject(components.securitySchemes) ? components.securitySchemes : {}; + const scheme = isJsonObject(schemes[OAUTH_SCHEME]) ? schemes[OAUTH_SCHEME] : {}; + const flows = isJsonObject(scheme.flows) ? scheme.flows : {}; + const flow = isJsonObject(flows.authorizationCode) ? flows.authorizationCode : {}; + const known = new Set([ + ...Object.keys(isJsonObject(flow.scopes) ? flow.scopes : {}), + ...Object.keys(scheme[GOOGLE_SCOPES] === undefined ? {} : objectAt(scheme[GOOGLE_SCOPES], `${at} ${OAUTH_SCHEME} ${GOOGLE_SCOPES}`)), + ]); + for (const [path, value] of Object.entries(objectAt(spec.paths, `${at} paths`))) + for (const { method, operation } of itemOperations(objectAt(value, `${at} ${path}`), `${at} ${path}`)) { + if (operation[GOOGLE_SCOPES] === undefined) continue; + const where = `${at} ${method} ${path}`; + const listed = scopeList(operation[GOOGLE_SCOPES], `${where} ${GOOGLE_SCOPES}`); + for (const scope of securityScopes(operation.security, `${where} security`)) + if (!listed.includes(scope)) + throw new Error(`${where}: ${GOOGLE_SCOPES} leaves out the security scope ${JSON.stringify(scope)}`); + for (const scope of listed) + if (!known.has(scope)) + throw new Error(`${where}: ${OAUTH_SCHEME} does not describe the listed scope ${JSON.stringify(scope)}`); + } +} + function singleServerUrl(servers: Json | undefined, at: string): string | undefined { if (servers === undefined) return undefined; if (!Array.isArray(servers) || servers.length !== 1)