diff --git a/fetcher.ts b/fetcher.ts index 36cc486f8..2d8535873 100644 --- a/fetcher.ts +++ b/fetcher.ts @@ -35,6 +35,22 @@ interface LaunchedCompany { question_answers: boolean; } +const loadExistingCompanies = async ( + failureReason: string +): Promise => { + try { + const data = await Deno.readTextFile("companies/all.json"); + return JSON.parse(data) as LaunchedCompany[]; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + const prefix = + error instanceof Deno.errors.NotFound + ? `No cached data available and ${failureReason}.` + : `Failed to load cached data after ${failureReason}.`; + throw new Error(`${prefix} ${message}`); + } +}; + const fetchAllCompanies = async (): Promise => { const baseUrl = "https://45bwzj1sgc-dsn.algolia.net/1/indexes/*/queries"; const params = new URLSearchParams({ @@ -46,17 +62,36 @@ const fetchAllCompanies = async (): Promise => { }); console.log("Fetching facets"); - const res = await fetch(`${baseUrl}?${params}`, { - method: "POST", - body: JSON.stringify({ - requests: [ - { - indexName, - params: `facets=%5B%22app_answers%22%2C%22app_video_public%22%2C%22batch%22%2C%22demo_day_video_public%22%2C%22highlight_black%22%2C%22highlight_latinx%22%2C%22highlight_women%22%2C%22industries%22%2C%22isHiring%22%2C%22nonprofit%22%2C%22question_answers%22%2C%22regions%22%2C%22subindustry%22%2C%22tags%22%2C%22top_company%22%5D&hitsPerPage=1000&maxValuesPerFacet=1000&query=&tagFilters=`, - }, - ], - }), - }); + let res: Response; + try { + res = await fetch(`${baseUrl}?${params}`, { + method: "POST", + body: JSON.stringify({ + requests: [ + { + indexName, + params: `facets=%5B%22app_answers%22%2C%22app_video_public%22%2C%22batch%22%2C%22demo_day_video_public%22%2C%22highlight_black%22%2C%22highlight_latinx%22%2C%22highlight_women%22%2C%22industries%22%2C%22isHiring%22%2C%22nonprofit%22%2C%22question_answers%22%2C%22regions%22%2C%22subindustry%22%2C%22tags%22%2C%22top_company%22%5D&hitsPerPage=1000&maxValuesPerFacet=1000&query=&tagFilters=`, + }, + ], + }), + }); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + console.warn( + `Failed to fetch facets. Using existing data. Error: ${message}` + ); + return loadExistingCompanies("facet fetch failed"); + } + + if (!res.ok) { + const errorText = await res.text(); + console.warn( + `Failed to fetch facets (${res.status} ${res.statusText}). Using existing data. Response: ${errorText}` + ); + return loadExistingCompanies( + `facet request returned ${res.status} ${res.statusText}` + ); + } const json = (await res.json()) as { results: { @@ -64,7 +99,16 @@ const fetchAllCompanies = async (): Promise => { facets: { batch: Record }; }[]; }; - const batches = json.results[0].facets.batch; + if (!json.results?.length) { + console.warn("No results returned. Using existing data."); + return loadExistingCompanies("response missing results"); + } + + const batches = json.results[0].facets?.batch; + if (!batches) { + console.warn("No batch facets returned. Using existing data."); + return loadExistingCompanies("response missing batch facets"); + } let allCompanies: LaunchedCompany[] = []; for (const [batch, count] of Object.entries(batches)) {