Sanitizing PII Before Vector Database Ingestion: Pinecone, Qdrant & Chroma

import { PrivacyScrubberEngine } from '@privacyscrubber/sdk';
import { Pinecone } from '@pinecone-database/pinecone';
import OpenAI from 'openai';

// 1. Initialize Zero-Trust in-memory redaction engine (<1ms latency)
const scrubber = new PrivacyScrubberEngine({
  profile: 'General',         // Support, DevOps, Financial, Legal, Healthcare profiles
  detectSecrets: true,        // Mask API keys, JWTs, and database URIs
  deterministic: true         // Same entity receives identical token across chunks
});

const openai = new OpenAI();
const pinecone = new Pinecone();
const index = pinecone.Index('enterprise-knowledge-base');

export async function ingestDocumentSafely(docId, rawContent, metadata = {}) {
  // 2. Sanitize raw text in volatile RAM BEFORE chunking & embedding
  const { scrubbedText, tokenMap } = scrubber.sanitize(rawContent);

  // 3. Generate embeddings exclusively on sanitized text
  const embeddingResponse = await openai.embeddings.create({
    model: 'text-embedding-3-small',
    input: scrubbedText
  });

  const vector = embeddingResponse.data[0].embedding;

  // 4. Upsert vector + sanitized metadata into Pinecone
  // ZERO cleartext PII is stored in vector space or metadata payloads!
  await index.upsert([
    {
      id: docId,
      values: vector,
      metadata: {
        ...metadata,
        sanitized_preview: scrubbedText.slice(0, 200),
        has_masked_entities: Object.keys(tokenMap).length > 0
      }
    }
  ]);

  return { docId, tokenCount: Object.keys(tokenMap).length };
}