Pinecone Vector Upsert PII Redaction: In-Memory Metadata Masking

import { Pinecone } from '@pinecone-database/pinecone';
import OpenAI from 'openai';
import { sanitize } from '@privacyscrubber/sdk';

const pc = new Pinecone();
const index = pc.index('customer-rag-index');
const openai = new OpenAI({ apiKey: process.env.OPENAI_API_KEY });

// 1. Raw incoming customer records with sensitive PII
const rawRecords = [
  {
    id: 'rec_01',
    text: 'Client Marcus Brody (SSN: 204-55-1982, Email: marcus@fintech.io) requested mortgage pre-approval for $450,000.',
    metadata: { author: 'Marcus Brody', department: 'Mortgage' }
  }
];

// 2. Sanitize text and metadata in local RAM before vectorization
const sanitizedVectors = await Promise.all(
  rawRecords.map(async (rec) => {
    const { scrubbedText } = sanitize(rec.text, {
      profile: 'Financial'
    });

    // Real OpenAI text embedding on sanitized text (zero cleartext PII sent to OpenAI)
    const embeddingRes = await openai.embeddings.create({
      model: 'text-embedding-3-small',
      input: scrubbedText
    });
    const embedding = embeddingRes.data[0].embedding;

    return {
      id: rec.id,
      values: embedding,
      metadata: {
        text: scrubbedText, // Contains [NAME_1], [SSN_1], [EMAIL_1]
        department: rec.metadata.department
      }
    };
  })
);

// 3. Upsert clean vectors to Pinecone
await index.upsert(sanitizedVectors);
console.log('Successfully upserted clean records to Pinecone with zero PII exposure!');