Busca Semântica com Embeddings em Next.js

-- Ativa pgvector. No Supabase, já vem disponível.
-- Em PostgreSQL local, instale: apt install postgresql-16-pgvector
CREATE EXTENSION IF NOT EXISTS vector;
 
CREATE TABLE documents (
  id SERIAL PRIMARY KEY,
  title TEXT NOT NULL,
  content TEXT NOT NULL,
  -- 1536 dimensões: compatível com text-embedding-3-small da OpenAI
  embedding vector(1536),
  metadata JSONB DEFAULT '{}',
  created_at TIMESTAMPTZ DEFAULT NOW()
);
 
-- IVFFlat é mais rápido para criar que HNSW, suficiente até ~100k vetores.
-- lists = sqrt(número de linhas). Para 10k docs, ~100 lists.
CREATE INDEX ON documents
  USING ivfflat (embedding vector_cosine_ops)
  WITH (lists = 100);

npm install openai pg pgvector

// src/lib/embeddings.ts
import OpenAI from "openai";
 
const openai = new OpenAI({
  apiKey: process.env.OPENAI_API_KEY,
});
 
// text-embedding-3-small custa $0.02/1M tokens, 5x mais barato que ada-002
// com qualidade comparável para busca semântica em português
const EMBEDDING_MODEL = "text-embedding-3-small";
 
export async function generateEmbedding(text: string): Promise<number[]> {
  // Limpa whitespace excessivo para não desperdiçar tokens
  const sanitized = text.replace(/\s+/g, " ").trim();
 
  const response = await openai.embeddings.create({
    model: EMBEDDING_MODEL,
    input: sanitized,
  });
 
  return response.data[0].embedding;
}
 
export async function generateEmbeddings(
  texts: string[]
): Promise<number[][]> {
  // A API aceita batch de até 2048 inputs por request.
  // Enviar em batch reduz latência vs. chamadas individuais.
  const response = await openai.embeddings.create({
    model: EMBEDDING_MODEL,
    input: texts.map((t) => t.replace(/\s+/g, " ").trim()),
  });
 
  return response.data.map((d) => d.embedding);
}

// src/app/api/documents/index/route.ts
import { NextRequest, NextResponse } from "next/server";
import { Pool } from "pg";
import pgvector from "pgvector/pg";
import { generateEmbedding } from "@/lib/embeddings";
 
const pool = new Pool({
  connectionString: process.env.DATABASE_URL,
});
 
// Registra o tipo vector uma vez no pool
pool.on("connect", async (client) => {
  await pgvector.registerType(client);
});
 
export async function POST(request: NextRequest) {
  const { title, content, metadata } = await request.json();
 
  if (!title || !content) {
    return NextResponse.json(
      { error: "title e content são obrigatórios" },
      { status: 400 }
    );
  }
 
  // Concatena título + conteúdo para o embedding capturar contexto completo.
  // Título sozinho perde nuance; conteúdo sozinho perde o resumo semântico.
  const textForEmbedding = `${title}\n\n${content}`;
  const embedding = await generateEmbedding(textForEmbedding);
 
  const client = await pool.connect();
  try {
    const result = await client.query(
      `INSERT INTO documents (title, content, embedding, metadata)
       VALUES ($1, $2, $3, $4)
       RETURNING id, title, created_at`,
      [title, content, pgvector.toSql(embedding), metadata ?? {}]
    );
 
    return NextResponse.json(result.rows[0], { status: 201 });
  } finally {
    client.release();
  }
}

// src/app/api/search/route.ts
import { NextRequest, NextResponse } from "next/server";
import { Pool } from "pg";
import pgvector from "pgvector/pg";
import { generateEmbedding } from "@/lib/embeddings";
 
const pool = new Pool({
  connectionString: process.env.DATABASE_URL,
});
 
pool.on("connect", async (client) => {
  await pgvector.registerType(client);
});
 
interface SearchResult {
  id: number;
  title: string;
  content: string;
  similarity: number;
}
 
export async function GET(request: NextRequest) {
  const query = request.nextUrl.searchParams.get("q");
  const limit = parseInt(request.nextUrl.searchParams.get("limit") ?? "5", 10);
 
  if (!query) {
    return NextResponse.json(
      { error: "Parâmetro q é obrigatório" },
      { status: 400 }
    );
  }
 
  const queryEmbedding = await generateEmbedding(query);
 
  const client = await pool.connect();
  try {
    // 1 - distância cosseno = similaridade (0 a 1, onde 1 é idêntico)
    // Filtra resultados com similaridade mínima para evitar lixo
    const { rows } = await client.query<SearchResult>(
      `SELECT
         id,
         title,
         LEFT(content, 300) AS content,
         1 - (embedding <=> $1) AS similarity
       FROM documents
       WHERE 1 - (embedding <=> $1) > 0.3
       ORDER BY embedding <=> $1
       LIMIT $2`,
      [pgvector.toSql(queryEmbedding), limit]
    );
 
    return NextResponse.json({ results: rows, query });
  } finally {
    client.release();
  }
}

// src/components/semantic-search.tsx
"use client";
 
import { useState, useCallback, useRef } from "react";
 
interface SearchResult {
  id: number;
  title: string;
  content: string;
  similarity: number;
}
 
export function SemanticSearch() {
  const [results, setResults] = useState<SearchResult[]>([]);
  const [loading, setLoading] = useState(false);
  const [query, setQuery] = useState("");
  const debounceRef = useRef<NodeJS.Timeout | null>(null);
 
  const search = useCallback(async (searchQuery: string) => {
    if (searchQuery.length < 3) {
      setResults([]);
      return;
    }
 
    setLoading(true);
    try {
      const response = await fetch(
        `/api/search?q=${encodeURIComponent(searchQuery)}&limit=5`
      );
      const data = await response.json();
      setResults(data.results ?? []);
    } finally {
      setLoading(false);
    }
  }, []);
 
  const handleChange = (value: string) => {
    setQuery(value);
    // 400ms de debounce: cada keystroke gera um embedding ($0.02/1M tokens),
    // mas a latência da API (~200ms) torna debounce menor que 300ms inútil
    if (debounceRef.current) clearTimeout(debounceRef.current);
    debounceRef.current = setTimeout(() => search(value), 400);
  };
 
  return (
    <div>
      <input
        type="search"
        value={query}
        onChange={(e) => handleChange(e.target.value)}
        placeholder="Buscar por significado..."
        aria-label="Campo de busca semântica"
      />
      {loading && <p>Buscando...</p>}
      <ul>
        {results.map((result) => (
          <li key={result.id}>
            <strong>{result.title}</strong>
            <span>
              {(result.similarity * 100).toFixed(1)}% de relevância
            </span>
            <p>{result.content}</p>
          </li>
        ))}
      </ul>
    </div>
  );
}

// ERRADO: expõe a API key da OpenAI no bundle do navegador
"use client";
 
import OpenAI from "openai";
 
const openai = new OpenAI({
  apiKey: "sk-proj-...", // Vai parar no bundle. Qualquer um extrai.
});
 
async function searchOnClient(query: string) {
  const embedding = await openai.embeddings.create({
    model: "text-embedding-3-small",
    input: query,
  });
  // ...
}

-- ERRADO: sem índice vetorial, busca vira sequential scan
CREATE TABLE documents (
  id SERIAL PRIMARY KEY,
  content TEXT,
  embedding JSONB  -- Funciona para guardar, mas não para buscar
);
 
-- Essa query precisa deserializar JSON, converter para array,
-- calcular distância manualmente. Em 10k docs, leva segundos.

-- CORRETO: tipo vector nativo com índice especializado
CREATE TABLE documents (
  id SERIAL PRIMARY KEY,
  content TEXT,
  embedding vector(1536)
);
 
CREATE INDEX ON documents USING ivfflat (embedding vector_cosine_ops) WITH (lists = 100);
 
-- Busca por similaridade usa o índice. Em 10k docs, < 15ms.

// ERRADO: artigo inteiro vira um vetor genérico
const embedding = await generateEmbedding(artigoCompleto); // 5000 palavras
 
// CORRETO: divide em chunks com overlap para preservar contexto
function chunkText(text: string, chunkSize = 500, overlap = 50): string[] {
  const words = text.split(/\s+/);
  const chunks: string[] = [];
 
  for (let i = 0; i < words.length; i += chunkSize - overlap) {
    chunks.push(words.slice(i, i + chunkSize).join(" "));
  }
 
  return chunks;
}