From LLM experiments
to products

1import { defer } from "@defer/client";
2import { createClient } from "@supabase/supabase-js";
3import { Configuration, OpenAIApi } from "openai";
4
5import { supabaseClient } from "./lib/supabase";
6import { openAiClient } from "./lib/openai";
7import { getRecentlyUpdatedDocuments } from "../lib";
8
9import refreshCategories from "./refreshCategories";
10
11async function generateEmbeddings(workspaceID: string) {
12    // 1. retrieve the newly created or updated documents from the database
13    const documents = await getRecentlyUpdatedDocuments(workspaceID);
14
15    for (const document of documents) {
16    // (OpenAI recommends replacing newlines with spaces for best results)
17    const input = document.replace(/\n/g, " ");
18
19    // 2. generate an embedding for each document
20    const embeddingResponse = await openAiClient.createEmbedding({
21        model: "text-embedding-ada-002",
22        input,
23    });
24
25    // 3. store the embedding as a vector in our Supabase Vector database
26    await supabaseClient.from("documents").insert({
27        content: document,
28        workspaceID,
29        embedding: embeddingResponse.data.data,
30    });
31    }
32
33    // 4. Enqueue another execution that will extract clusters
34    //   to refresh categories
35    //   (see https://cookbook.openai.com/examples/clustering)
36    await refreshCategories(workspaceID);
37}
38
39export default defer(generateEmbeddings, {
40    concurrency: 5, // limit concurrency for OpenAI rate limiting
41    retry: 2, // enable retry to handle external failures
42});
43