Pinecone Vector Upsert PII Redaction: In-Memory Metadata Masking

import { Pinecone } from '@pinecone-database/pinecone';
import { scrubText } from '@privacyscrubber/sdk';

const pc = new Pinecone();
const index = pc.index('customer-rag-index');

// 1. Raw incoming customer records with sensitive PII
const rawRecords = [
  {
    id: 'rec_01',
    text: 'Client Marcus Brody (SSN: 204-55-1982, Email: marcus@fintech.io) requested mortgage pre-approval for $450,000.',
    metadata: { author: 'Marcus Brody', department: 'Mortgage' }
  }
];

// 2. Sanitize text and metadata in local RAM before vectorization
const sanitizedVectors = await Promise.all(
  rawRecords.map(async (rec) => {
    const { sanitizedText } = scrubText(rec.text, {
      profile: 'Financial',
      preserveFormatting: true
    });

    // Mock embedding generation on sanitized text (e.g. OpenAI text-embedding-3-small)
    const embedding = new Array(1536).fill(0.01);

    return {
      id: rec.id,
      values: embedding,
      metadata: {
        text: sanitizedText, // Contains [NAME_1], [SSN_1], [EMAIL_1]
        department: rec.metadata.department
      }
    };
  })
);

// 3. Upsert clean vectors to Pinecone
await index.upsert(sanitizedVectors);
console.log('Successfully upserted clean records to Pinecone with zero PII exposure!');