Dense Text Embeddings

import { Pinecone } from '@pinecone-client/web';
import OpenAI from 'openai';

const openai = new OpenAI();
const pinecone = new Pinecone();

// Embed once at full 3,072 dims
const response = await openai.embeddings.create({
  model: 'text-embedding-3-large',
  input: 'The quick brown fox jumps over the lazy dog',
  dimensions: 3072, // Store full
});

let vector = response.data[0].embedding;

// Upsert full to vector DB
await pinecone.index('main').upsert([{
  id: 'doc-1',
  values: vector,
  metadata: { text: '...' },
}]);

// At query time, truncate to smaller dimension for speed/cost
const queryResponse = await openai.embeddings.create({
  model: 'text-embedding-3-large',
  input: 'fox jumps',
  dimensions: 256, // Query with smaller dim
});

let queryVector = queryResponse.data[0].embedding.slice(0, 256);

// Search: Pinecone still uses full 3,072 stored dims
// but query vector is smaller (faster, lower latency)
const results = await pinecone.index('main').query({
  vector: queryVector,
  topK: 10,
});

import { batch } from 'lodash';
import pRetry from 'p-retry';
import OpenAI from 'openai';

const openai = new OpenAI({ apiKey: process.env.OPENAI_API_KEY });

async function embedBatch(texts: string[], batchSize: number = 100) {
  const batches = batch(texts, batchSize);
  const results: number[][] = [];

  for (const batchTexts of batches) {
    const embedded = await pRetry(
      () =>
        openai.embeddings.create({
          model: 'text-embedding-3-large',
          input: batchTexts,
          dimensions: 1536, // Use Matryoshka for cost
        }),
      {
        retries: 3,
        onFailedAttempt: (error) => {
          console.warn(
            `Embedding batch attempt ${error.attemptNumber} failed.`,
            error.message
          );
        },
      }
    );

    const sortedResults = embedded.data.sort((a, b) => a.index - b.index);
    results.push(...sortedResults.map((r) => r.embedding));

    // Rate limit: OpenAI allows 500K tokens/min
    await new Promise((resolve) => setTimeout(resolve, 100));
  }

  return results;
}

Cost = (1,000,000 docs × 500 tokens/doc) × ($0.02 / 1M tokens) = $10
Matryoshka cost at query time:
- 10,000 queries × 200 tokens = 2M tokens × $0.02 / 1M = $0.04/month

Embedding time: 100K docs / 1000 tokens-per-sec ≈ 50 seconds ≈ $0.003
Subsequent queries: ~free (GPU already rented)

import * as fs from 'fs';
import OpenAI from 'openai';
import { batch } from 'lodash';

async function embedLargeDataset(
  inputFile: string,
  outputFile: string,
  batchSize: number = 100
) {
  const lines = fs
    .readFileSync(inputFile, 'utf-8')
    .split('\n')
    .filter((l) => l.trim());

  const batches = batch(lines, batchSize);
  const client = new OpenAI();
  const results: Array<{ text: string; embedding: number[] }> = [];

  for (let i = 0; i < batches.length; i++) {
    const batchTexts = batches[i];
    const response = await client.embeddings.create({
      model: 'text-embedding-3-large',
      input: batchTexts,
      dimensions: 512, // Matryoshka: small dims for cost
    });

    const sorted = response.data.sort((a, b) => a.index - b.index);
    sorted.forEach((item, idx) => {
      results.push({
        text: batchTexts[idx],
        embedding: item.embedding,
      });
    });

    console.log(`Progress: ${((i + 1) / batches.length) * 100}%`);
  }

  fs.writeFileSync(outputFile, JSON.stringify(results, null, 2));
  console.log(`Embedded ${results.length} texts to ${outputFile}`);
}

// Usage
await embedLargeDataset('documents.txt', 'embeddings.json', 100);