processAndInsertEmbeddings() — supabase Function Reference

Source Code

apps/docs/scripts/search/generate-embeddings.ts lines 252–316

async function processAndInsertEmbeddings(
  openai: OpenAI,
  supabaseClient: SupabaseClient,
  pageSectionTable: string,
  allSections: PageSectionForEmbedding[],
  pageInfoMap: Map<number, PageInfo>
): Promise<ProcessingResult> {
  const result: ProcessingResult = {
    successfulPages: new Set(),
    failedPages: new Set(),
    totalSectionsProcessed: 0,
    totalSectionsInserted: 0,
  }

  if (allSections.length === 0) {
    return result
  }

  console.log(`Processing ${allSections.length} sections with embeddings + insertion`)

  const embeddingBatches = createBatches(allSections, CONFIG.OPENAI_BATCH_SIZE)

  // Track sections inserted per page
  const pageSectionsInserted = new Map<number, number>()

  for (let batchIndex = 0; batchIndex < embeddingBatches.length; batchIndex++) {
    const batch = embeddingBatches[batchIndex]
    try {
      const batchResult = await processEmbeddingBatch(
        openai,
        batch,
        batchIndex,
        embeddingBatches.length
      )

      result.totalSectionsProcessed += batchResult.processedCount

      if (batchResult.sectionsWithEmbeddings.length > 0) {
        const insertedCount = await insertSectionBatch(
          supabaseClient,
          pageSectionTable,
          batchResult.sectionsWithEmbeddings
        )
        result.totalSectionsInserted += insertedCount
        updatePageInsertionCounts(pageSectionsInserted, batchResult.sectionsWithEmbeddings)
      }

      // Mark failed section pages
      batchResult.failedSectionIndexes.forEach((i) => {
        result.failedPages.add(batch[i].pageId)
      })
    } catch (error) {
      console.error(`Batch ${batchIndex + 1} completely failed:`, error)
      batch.forEach((section) => result.failedPages.add(section.pageId))
    }

    if (batchIndex < embeddingBatches.length - 1) {
      await delay(CONFIG.OPENAI_BASE_DELAY_MS)
    }
  }

  computePageResults(pageInfoMap, pageSectionsInserted, result)

  return result
}