prepareSections() — supabase Function Reference

Architecture documentation for the prepareSections() function in generate-embeddings.ts from the supabase codebase.

Function typescript DocumentationSystem SearchEmbeddings calls 2 called by 1

Entity Profile

DocumentationSystem→ SearchEmbeddings→ prepareSections() — supabase Function Reference

Dependency Diagram

graph TD
  e3264b04_c226_7750_2974_b29b0dc6cd3a["prepareSections()"]
  b280bae5_b0f6_6775_91cb_d0697bd4d074["generateEmbeddings()"]
  b280bae5_b0f6_6775_91cb_d0697bd4d074 -->|calls| e3264b04_c226_7750_2974_b29b0dc6cd3a
  bb2c79de_bc44_4879_8e44_cbf11b580212["fetchAllSources()"]
  e3264b04_c226_7750_2974_b29b0dc6cd3a -->|calls| bb2c79de_bc44_4879_8e44_cbf11b580212
  94c6fd9e_4273_b489_8fb9_0054d6d1c1ce["createBatches()"]
  e3264b04_c226_7750_2974_b29b0dc6cd3a -->|calls| 94c6fd9e_4273_b489_8fb9_0054d6d1c1ce
  style e3264b04_c226_7750_2974_b29b0dc6cd3a fill:#6366f1,stroke:#818cf8,color:#fff

Relationship Graph

Source Code

apps/docs/scripts/search/generate-embeddings.ts lines 123–250

async function prepareSections(
  supabaseClient: SupabaseClient,
  pageTable: string,
  pageSectionTable: string,
  shouldRefresh: boolean,
  refreshVersion: string,
  refreshDate: Date,
  fullIndex = true,
  debug = false
): Promise<PreparedSections> {
  const embeddingSources = await fetchAllSources(fullIndex)
  console.log(`Discovered ${embeddingSources.length} sources`)

  const allSectionsToProcess: PageSectionForEmbedding[] = []
  const pageInfoMap = new Map<number, PageInfo>()

  for (const sourceBatch of createBatches(embeddingSources, CONFIG.SOURCE_CONCURRENCY)) {
    await Promise.all(
      sourceBatch.map(async (embeddingSource) => {
        const { type, source, path } = embeddingSource

        try {
          const {
            checksum,
            sections,
            meta = {},
            ragIgnore = false,
          }: {
            checksum: string
            sections: Section[]
            ragIgnore?: boolean
            meta?: Record<string, unknown>
          } = await embeddingSource.process()

          const { error: fetchPageError, data: existingPage } = await supabaseClient
            .from(pageTable)
            .select('id, path, checksum')
            .filter('path', 'eq', path)
            .limit(1)
            .maybeSingle()

          if (fetchPageError) throw fetchPageError

          if (!shouldRefresh && existingPage?.checksum === checksum) {
            const { error: updatePageError } = await supabaseClient
              .from(pageTable)
              .update({
                type,
                source,
                meta,
                version: refreshVersion,
                last_refresh: refreshDate,
              })
              .filter('id', 'eq', existingPage.id)
            if (updatePageError) throw updatePageError
            return
          }

          if (existingPage) {
            if (debug) {
              console.log(
                !shouldRefresh
                  ? `[${path}] Docs have changed, removing old page sections and their embeddings`
                  : `[${path}] Refresh flag set, removing old page sections and their embeddings`
              )
            }

            const { error: deletePageSectionError } = await supabaseClient
              .from(pageSectionTable)
              .delete()
              .filter('page_id', 'eq', existingPage.id)
            if (deletePageSectionError) throw deletePageSectionError
          }

          const { error: upsertPageError, data: page } = await supabaseClient
            .from(pageTable)
            .upsert(
              {
                checksum: null,
                path,
                type,
                source,
                meta,
                content: embeddingSource.extractIndexedContent(),
                version: refreshVersion,
                last_refresh: refreshDate,
              },
              { onConflict: 'path' }
            )
            .select()
            .limit(1)
            .single()
          if (upsertPageError) throw upsertPageError

          if (debug) {
            console.log(`[${path}] Preparing ${sections.length} page sections for processing`)
          }

          pageInfoMap.set(page.id, {
            pageId: page.id,
            path,
            checksum,
            sectionsCount: sections.length,
          })

          const sectionsForBatching = sections.map(({ slug, heading, content }) => ({
            pageId: page.id,
            path,
            slug,
            heading,
            content,
            input: content.replace(/\n/g, ' '),
            ragIgnore,
          }))
          allSectionsToProcess.push(...sectionsForBatching)
        } catch (err) {
          console.error(`Error preparing path '${path}' for processing.`)
          console.error(err)
        }
      })
    )
  }

  console.log(
    `Prepared ${allSectionsToProcess.length} sections for processing from ${pageInfoMap.size} pages`
  )
  return { allSectionsToProcess, pageInfoMap }
}