generateEmbeddings() — supabase Function Reference

Source Code

apps/docs/scripts/search/generate-embeddings.ts lines 439–521

async function generateEmbeddings() {
  const shouldRefresh = Boolean(args.values.refresh)
  const debug = Boolean(args.values.debug)

  const nimbus = isNimbusMode()
  if (nimbus) {
    console.log('Running in Nimbus mode - will filter content based on disabled feature flags')
  }

  requireEnvOrThrow([
    'DOCS_GITHUB_APP_ID',
    'DOCS_GITHUB_APP_INSTALLATION_ID',
    'DOCS_GITHUB_APP_PRIVATE_KEY',
    'NEXT_PUBLIC_MISC_ANON_KEY',
    'NEXT_PUBLIC_MISC_URL',
    'NEXT_PUBLIC_SUPABASE_URL',
    'OPENAI_API_KEY',
    'SUPABASE_SECRET_KEY',
  ])

  const supabaseClient = initSupabase()

  const refreshVersion = uuidv4()
  const refreshDate = new Date()

  const { pageTable, pageSectionTable } = getPageTables()
  const openai = new OpenAI({ apiKey: process.env.OPENAI_API_KEY })

  console.log(
    shouldRefresh
      ? 'Refresh flag set, re-generating all pages'
      : 'Checking which pages are new or have changed'
  )

  const { allSectionsToProcess, pageInfoMap } = await prepareSections(
    supabaseClient,
    pageTable,
    pageSectionTable,
    shouldRefresh,
    refreshVersion,
    refreshDate,
    !nimbus,
    debug
  )

  let processingResult: ProcessingResult
  try {
    processingResult = await processAndInsertEmbeddings(
      openai,
      supabaseClient,
      pageSectionTable,
      allSectionsToProcess,
      pageInfoMap
    )
    console.log(
      `Processing complete: ${processingResult.totalSectionsInserted}/${processingResult.totalSectionsProcessed} sections inserted successfully`
    )
    console.log(
      `Page summary: ${processingResult.successfulPages.size} successful, ${processingResult.failedPages.size} failed`
    )
  } catch (error) {
    console.error('Critical error during embedding processing:', error)
    console.log('Exiting due to complete processing failure')
    return
  }

  console.log(`\nUpdating checksums for ${processingResult.successfulPages.size} successful pages`)
  const successfulChecksumUpdates = await updateSuccessfulChecksums(
    supabaseClient,
    pageTable,
    pageInfoMap,
    processingResult
  )
  console.log(
    `Successfully updated checksums for ${successfulChecksumUpdates}/${processingResult.successfulPages.size} successful pages`
  )

  logFailedPages(pageInfoMap, processingResult)

  await purgeOldPages(supabaseClient, pageTable, refreshVersion)

  console.log('Embedding generation complete')
}