improvement(confluence): stream paginated space selector results

waleedlatif1 · waleedlatif1 · commit a677ae851e88 · 2026-05-05T19:58:57.000-07:00
Bake pagination support into the selector abstraction via an opt-in
fetchPage definition so dropdowns populate progressively instead of
blocking on a full page-walk. Confluence spaces now stream current
then archived in a single cursor sequence.
diff --git a/apps/sim/app/api/tools/confluence/selector-spaces/route.ts b/apps/sim/app/api/tools/confluence/selector-spaces/route.ts
@@ -19,13 +19,33 @@ const logger = createLogger('ConfluenceSelectorSpacesAPI')
 
 export const dynamic = 'force-dynamic'
 
+const PAGE_LIMIT = 250
+
+type SpaceStatus = 'current' | 'archived'
+
+/**
+ * Cursor format: `<status>:<innerCursor>`. Empty inner cursor means "first page
+ * of that status". When current is exhausted we hand back `archived:` so the
+ * client transparently flips to the archived stream — listing both surfaces
+ * archived spaces in the dropdown, which would otherwise only be reachable by
+ * typing the space key manually even though sync works against archived spaces.
+ */
+function parseCursor(raw: string | undefined): { status: SpaceStatus; inner?: string } {
+  if (!raw) return { status: 'current' }
+  const idx = raw.indexOf(':')
+  if (idx === -1) return { status: 'current' }
+  const status = raw.slice(0, idx) === 'archived' ? 'archived' : 'current'
+  const inner = raw.slice(idx + 1)
+  return { status, inner: inner || undefined }
+}
+
 export const POST = withRouteHandler(async (request: NextRequest) => {
   const requestId = generateRequestId()
   try {
     const parsed = await parseRequest(confluenceSpacesSelectorContract, request, {})
     if (!parsed.success) return parsed.response
 
-    const { credential, workflowId, domain } = parsed.data.body
+    const { credential, workflowId, domain, cursor } = parsed.data.body
 
     if (!credential) {
       logger.error('Missing credential in request')
@@ -44,11 +64,6 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
       return NextResponse.json({ error: authz.error || 'Unauthorized' }, { status: 403 })
     }
 
-    // Resolve once so we know whether this is an Atlassian SA credential before
-    // doing any token / cloudId work. Atlassian SAs short-circuit the entire path:
-    // the API token IS the access token, and cloudId lives in the encrypted secret —
-    // so we skip refreshAccessTokenIfNeeded (avoids a redundant resolve+decrypt) and
-    // skip getConfluenceCloudId (which 401s for scoped SA tokens).
     const resolved = await resolveOAuthAccountId(credential)
     const isAtlassianServiceAccount =
       resolved?.providerId === ATLASSIAN_SERVICE_ACCOUNT_PROVIDER_ID && !!resolved.credentialId
@@ -84,82 +99,50 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
     }
 
     const baseUrl = `https://api.atlassian.com/ex/confluence/${cloudIdValidation.sanitized}/wiki/api/v2/spaces`
-    const PAGE_LIMIT = 250
-    const MAX_PAGES = 20
-
-    /**
-     * Confluence v2 `/spaces` defaults to `status=current` and treats `status`
-     * as a single-value enum, so archived spaces never surface from one call.
-     * Listing both surfaces archived spaces in the dropdown — they would
-     * otherwise only be reachable by typing the space key manually, even
-     * though sync works against archived spaces just fine.
-     */
-    async function fetchAllPages(status: 'current' | 'archived'): Promise<{
-      spaces: { id: string; name: string; key: string; status: string }[]
-      capped: boolean
-    }> {
-      const collected: { id: string; name: string; key: string; status: string }[] = []
-      let cursor: string | undefined
-      let pageCount = 0
-
-      while (pageCount < MAX_PAGES) {
-        const params = new URLSearchParams({ limit: String(PAGE_LIMIT), status })
-        if (cursor) params.set('cursor', cursor)
-        const url = `${baseUrl}?${params.toString()}`
-
-        const response = await fetch(url, {
-          method: 'GET',
-          headers: { Accept: 'application/json', Authorization: `Bearer ${accessToken}` },
-        })
+    const { status, inner } = parseCursor(cursor)
 
-        if (!response.ok) {
-          const errorText = await response.text()
-          throw new Error(
-            parseAtlassianErrorMessage(response.status, response.statusText, errorText)
-          )
-        }
-
-        const data = await response.json()
-        for (const space of data.results || []) {
-          collected.push({ id: space.id, name: space.name, key: space.key, status })
-        }
-
-        const nextLink = data._links?.next as string | undefined
-        if (!nextLink) return { spaces: collected, capped: false }
-        try {
-          cursor = new URL(nextLink, 'https://placeholder').searchParams.get('cursor') || undefined
-        } catch {
-          cursor = undefined
-        }
-        if (!cursor) return { spaces: collected, capped: false }
-        pageCount += 1
-      }
+    const params = new URLSearchParams({ limit: String(PAGE_LIMIT), status })
+    if (inner) params.set('cursor', inner)
+    const url = `${baseUrl}?${params.toString()}`
 
-      return { spaces: collected, capped: true }
+    const response = await fetch(url, {
+      method: 'GET',
+      headers: { Accept: 'application/json', Authorization: `Bearer ${accessToken}` },
+    })
+
+    if (!response.ok) {
+      const errorText = await response.text()
+      const message = parseAtlassianErrorMessage(response.status, response.statusText, errorText)
+      logger.error('Confluence API error response', { error: message, status: response.status })
+      return NextResponse.json({ error: message }, { status: 502 })
     }
 
-    let currentResult: Awaited<ReturnType<typeof fetchAllPages>>
-    let archivedResult: Awaited<ReturnType<typeof fetchAllPages>>
-    try {
-      ;[currentResult, archivedResult] = await Promise.all([
-        fetchAllPages('current'),
-        fetchAllPages('archived'),
-      ])
-    } catch (error) {
-      logger.error('Confluence API error response', { error: (error as Error).message })
-      return NextResponse.json({ error: (error as Error).message }, { status: 502 })
+    const data = await response.json()
+    const spaces = (data.results || []).map((space: { id: string; name: string; key: string }) => ({
+      id: space.id,
+      name: space.name,
+      key: space.key,
+      status,
+    }))
+
+    let nextInner: string | undefined
+    const nextLink = data._links?.next as string | undefined
+    if (nextLink) {
+      try {
+        nextInner = new URL(nextLink, 'https://placeholder').searchParams.get('cursor') || undefined
+      } catch {
+        nextInner = undefined
+      }
     }
 
-    if (currentResult.capped || archivedResult.capped) {
-      logger.warn('Confluence space listing hit pagination cap', {
-        cap: MAX_PAGES * PAGE_LIMIT,
-        currentCount: currentResult.spaces.length,
-        archivedCount: archivedResult.spaces.length,
-      })
+    let nextCursor: string | undefined
+    if (nextInner) {
+      nextCursor = `${status}:${nextInner}`
+    } else if (status === 'current') {
+      nextCursor = 'archived:'
     }
 
-    const spaces = [...currentResult.spaces, ...archivedResult.spaces]
-    return NextResponse.json({ spaces })
+    return NextResponse.json({ spaces, nextCursor })
   } catch (error) {
     logger.error('Error listing Confluence spaces:', error)
     return NextResponse.json(
diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/selector-combobox/selector-combobox.tsx b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/selector-combobox/selector-combobox.tsx
@@ -53,6 +53,7 @@ export function SelectorCombobox({
   const {
     data: options = [],
     isLoading,
+    hasMore,
     error,
   } = useSelectorOptions(selectorKey, {
     context: selectorContext,
@@ -67,6 +68,7 @@ export function SelectorCombobox({
     Boolean(activeValue) &&
     Boolean(missingOptionLabel) &&
     !isLoading &&
+    !hasMore &&
     !optionMap.get(activeValue!)
   const selectedLabel = activeValue
     ? hasMissingOption
diff --git a/apps/sim/hooks/selectors/providers/confluence/selectors.ts b/apps/sim/hooks/selectors/providers/confluence/selectors.ts
@@ -22,21 +22,53 @@ export const confluenceSelectors = {
     ],
     enabled: ({ context }) => Boolean(context.oauthCredential && context.domain),
     fetchList: async ({ context, signal }: SelectorQueryArgs) => {
+      const credentialId = ensureCredential(context, 'confluence.spaces')
+      const domain = ensureDomain(context, 'confluence.spaces')
+      const collected: { id: string; label: string }[] = []
+      let cursor: string | undefined
+      do {
+        const data = await requestJson(selectorContracts.confluenceSpacesSelectorContract, {
+          body: {
+            credential: credentialId,
+            workflowId: context.workflowId,
+            domain,
+            cursor,
+          },
+          signal,
+        })
+        for (const space of data.spaces || []) {
+          collected.push({ id: space.id, label: formatConfluenceSpaceLabel(space) })
+        }
+        cursor = data.nextCursor
+      } while (cursor)
+      return collected
+    },
+    fetchPage: async ({ context, cursor, signal }) => {
       const credentialId = ensureCredential(context, 'confluence.spaces')
       const domain = ensureDomain(context, 'confluence.spaces')
       const data = await requestJson(selectorContracts.confluenceSpacesSelectorContract, {
         body: {
           credential: credentialId,
           workflowId: context.workflowId,
           domain,
+          cursor,
         },
         signal,
       })
-      return (data.spaces || []).map((space) => ({
-        id: space.id,
-        label: formatConfluenceSpaceLabel(space),
-      }))
+      return {
+        items: (data.spaces || []).map((space) => ({
+          id: space.id,
+          label: formatConfluenceSpaceLabel(space),
+        })),
+        nextCursor: data.nextCursor,
+      }
     },
+    /**
+     * Resolves a single space label. Hits only the first page — the dropdown's
+     * `fetchPage` stream populates the options cache for spaces beyond page 1,
+     * and `useSelectorOptionMap` merges them in. Walking all pages here would
+     * double API load since the stream is already running in parallel.
+     */
     fetchById: async ({ context, detailId, signal }: SelectorQueryArgs) => {
       if (!detailId) return null
       const credentialId = ensureCredential(context, 'confluence.spaces')
diff --git a/apps/sim/hooks/selectors/types.ts b/apps/sim/hooks/selectors/types.ts
@@ -100,11 +100,26 @@ export interface SelectorQueryArgs {
   signal?: AbortSignal
 }
 
+export interface SelectorPage {
+  items: SelectorOption[]
+  nextCursor?: string
+}
+
+export interface SelectorPageArgs extends SelectorQueryArgs {
+  cursor?: string
+}
+
 export interface SelectorDefinition {
   key: SelectorKey
   contracts?: readonly AnyApiRouteContract[]
   getQueryKey: (args: SelectorQueryArgs) => QueryKey
   fetchList: (args: SelectorQueryArgs) => Promise<SelectorOption[]>
+  /**
+   * Optional. When defined, the selector hook fetches one page at a time and
+   * auto-drains remaining pages so the dropdown populates progressively.
+   * Returns `{ items, nextCursor }`; `nextCursor: undefined` ends the stream.
+   */
+  fetchPage?: (args: SelectorPageArgs) => Promise<SelectorPage>
   fetchById?: (args: SelectorQueryArgs) => Promise<SelectorOption | null>
   enabled?: (args: SelectorQueryArgs) => boolean
   staleTime?: number
diff --git a/apps/sim/hooks/selectors/use-selector-query.ts b/apps/sim/hooks/selectors/use-selector-query.ts
@@ -1,30 +1,110 @@
-import { useMemo } from 'react'
-import { useQuery } from '@tanstack/react-query'
+import { useEffect, useMemo } from 'react'
+import { useInfiniteQuery, useQuery } from '@tanstack/react-query'
 import { extractEnvVarName, isEnvVarReference, isReference } from '@/executor/constants'
 import { usePersonalEnvironment } from '@/hooks/queries/environment'
 import { getSelectorDefinition, mergeOption } from '@/hooks/selectors/registry'
-import type { SelectorKey, SelectorOption, SelectorQueryArgs } from '@/hooks/selectors/types'
+import type {
+  SelectorKey,
+  SelectorOption,
+  SelectorPage,
+  SelectorQueryArgs,
+} from '@/hooks/selectors/types'
 
 interface SelectorHookArgs extends Omit<SelectorQueryArgs, 'key'> {
   search?: string
   detailId?: string
   enabled?: boolean
 }
 
-export function useSelectorOptions(key: SelectorKey, args: SelectorHookArgs) {
+export interface SelectorOptionsResult {
+  data: SelectorOption[] | undefined
+  isLoading: boolean
+  isFetching: boolean
+  /**
+   * True while paginated selectors are draining remaining pages in the
+   * background. Always false for non-paginated selectors.
+   */
+  isFetchingMore: boolean
+  /**
+   * True when the paginated selector still has more pages queued. Always false
+   * for non-paginated selectors.
+   */
+  hasMore: boolean
+  error: Error | null
+}
+
+const EMPTY_PAGE: SelectorPage = { items: [], nextCursor: undefined }
+
+export function useSelectorOptions(
+  key: SelectorKey,
+  args: SelectorHookArgs
+): SelectorOptionsResult {
   const definition = getSelectorDefinition(key)
   const queryArgs: SelectorQueryArgs = {
     key,
     context: args.context,
     search: args.search,
   }
   const isEnabled = args.enabled ?? (definition.enabled ? definition.enabled(queryArgs) : true)
-  return useQuery<SelectorOption[]>({
+  const supportsPagination = Boolean(definition.fetchPage)
+
+  const flatQuery = useQuery<SelectorOption[]>({
     queryKey: definition.getQueryKey(queryArgs),
     queryFn: ({ signal }) => definition.fetchList({ ...queryArgs, signal }),
-    enabled: isEnabled,
+    enabled: !supportsPagination && isEnabled,
+    staleTime: definition.staleTime ?? 30_000,
+  })
+
+  const pagedQuery = useInfiniteQuery<SelectorPage>({
+    queryKey: [...definition.getQueryKey(queryArgs), 'paged'],
+    queryFn: ({ pageParam, signal }) => {
+      if (!definition.fetchPage) return Promise.resolve(EMPTY_PAGE)
+      return definition.fetchPage({
+        ...queryArgs,
+        cursor: pageParam as string | undefined,
+        signal,
+      })
+    },
+    getNextPageParam: (last) => last.nextCursor,
+    initialPageParam: undefined as string | undefined,
+    enabled: supportsPagination && isEnabled,
     staleTime: definition.staleTime ?? 30_000,
   })
+
+  const { hasNextPage, isFetchingNextPage, fetchNextPage, isError } = pagedQuery
+  useEffect(() => {
+    if (!supportsPagination) return
+    if (isError) return
+    if (hasNextPage && !isFetchingNextPage) {
+      void fetchNextPage()
+    }
+  }, [supportsPagination, hasNextPage, isFetchingNextPage, isError, fetchNextPage])
+
+  const pagedOptions = useMemo<SelectorOption[] | undefined>(() => {
+    if (!supportsPagination) return undefined
+    if (!pagedQuery.data) return undefined
+    return pagedQuery.data.pages.flatMap((page) => page.items)
+  }, [supportsPagination, pagedQuery.data])
+
+  if (supportsPagination) {
+    return {
+      data: pagedOptions,
+      isLoading: pagedQuery.isLoading,
+      isFetching: pagedQuery.isFetching,
+      isFetchingMore: pagedQuery.isFetchingNextPage,
+      hasMore: pagedQuery.hasNextPage ?? false,
+      error: (pagedQuery.error as Error | null) ?? null,
+    }
+  }
+
+  return {
+    data: flatQuery.data,
+    isLoading: flatQuery.isLoading,
+    isFetching: flatQuery.isFetching,
+    isFetchingMore: false,
+    hasMore: false,
+    error: (flatQuery.error as Error | null) ?? null,
+  }
 }
 
 export function useSelectorOptionDetail(
diff --git a/apps/sim/lib/api/contracts/selectors/confluence.ts b/apps/sim/lib/api/contracts/selectors/confluence.ts
