EU AI Act Article 10 Training Data Filter in Node.js (<1ms)

import fs from 'fs';
import readline from 'readline';
import { scrubText, unscrubText } from '@privacyscrubber/sdk';

interface TrainingSample {
  id: string;
  instruction: string;
  input: string;
  output: string;
}

async function sanitizeTrainingDataset(inputJsonlPath: string, outputJsonlPath: string) {
  const fileStream = fs.createReadStream(inputJsonlPath);
  const rl = readline.createInterface({ input: fileStream, crlfDelay: Infinity });
  const outputStream = fs.createWriteStream(outputJsonlPath, { flags: 'w' });

  let recordCount = 0;
  let totalMaskedEntities = 0;
  const startTime = Date.now();

  for await (const line of rl) {
    if (!line.trim()) continue;
    const record: TrainingSample = JSON.parse(line);

    // 1. Scrub prompt input and completion output in local V8 heap memory (<1ms)
    const scrubbedInput = scrubText(record.input, {
      profile: 'General',
      preserveFormatting: true
    });

    const scrubbedOutput = scrubText(record.output, {
      profile: 'General',
      preserveFormatting: true
    });

    // 2. Write sanitized training pair with deterministic synthetic tokens
    const cleanRecord = {
      id: record.id,
      instruction: record.instruction,
      input: scrubbedInput.sanitizedText,
      output: scrubbedOutput.sanitizedText,
      ztdsArticle10Verified: true
    };

    outputStream.write(JSON.stringify(cleanRecord) + '\n');
    recordCount++;
    totalMaskedEntities += Object.keys(scrubbedInput.sessionMap).length + Object.keys(scrubbedOutput.sessionMap).length;
  }

  outputStream.end();
  const durationMs = Date.now() - startTime;
  console.log(`Processed ${recordCount} training pairs in ${durationMs}ms (${(durationMs / recordCount).toFixed(2)}ms/record).`);
  console.log(`Masked ${totalMaskedEntities} high-risk PII entities. Clean dataset saved to ${outputJsonlPath}.`);
}

// Example execution on fine-tuning dataset
sanitizeTrainingDataset('raw_customer_conversations.jsonl', 'sanitized_article10_clean.jsonl');