Text Extraction

async function extractCitationsFromText(text) {
  const citations = [];

  // Pattern 1: APA-style in-text citations (Author, Year)
  const apaPattern = /\(([A-Z][a-z]+(?:\s+(?:&|and)\s+[A-Z][a-z]+)?),\s+(\d{4}[a-z]?)\)/g;
  let match;

  while ((match = apaPattern.exec(text)) !== null) {
    citations.push({
      authors: match[1],
      year: match[2],
      style: 'apa',
      context: extractContext(text, match.index, 100)
    });
  }

  // Pattern 2: Reference section entries
  const refSection = extractReferenceSection(text);
  if (refSection) {
    const refCitations = parseReferenceSection(refSection);
    citations.push(...refCitations);
  }

  // Pattern 3: DOI extraction
  const doiPattern = /10\.\d{4,}\/[^\s]+/g;
  const dois = [...new Set(text.match(doiPattern) || [])];

  // Enrich citations with DOIs
  enrichCitationsWithDOIs(citations, dois);

  return deduplicateCitations(citations);
}

function extractReferenceSection(text) {
  const patterns = [
    /References\s+([\s\S]+?)(?=\n\n[A-Z]|\Z)/i,
    /Bibliography\s+([\s\S]+?)(?=\n\n[A-Z]|\Z)/i,
    /Works Cited\s+([\s\S]+?)(?=\n\n[A-Z]|\Z)/i
  ];

  for (const pattern of patterns) {
    const match = text.match(pattern);
    if (match) return match[1];
  }

  return null;
}

function parseReferenceEntry(entry) {
  const patterns = {
    // APA: Author(s). (Year). Title. Journal.
    apa: /^([^(]+)\((\d{4})\)\.\s+([^.]+)\.\s+([^.]+)/,

    // MLA: Author. "Title" Publication, Year.
    mla: /^([^.]+)\.\s+"([^"]+)"\s+([^,]+),\s+(\d{4})/,

    // Chicago: Author. Title. Publisher, Year.
    chicago: /^([^.]+)\.\s+([^.]+)\.\s+([^:]+):\s+([^,]+),\s+(\d{4})/
  };

  for (const [style, pattern] of Object.entries(patterns)) {
    const match = entry.match(pattern);
    if (match) {
      return {
        rawEntry: entry,
        authors: match[1].trim(),
        year: match[2],
        title: match[3]?.trim(),
        publication: match[4]?.trim(),
        style,
        type: 'reference'
      };
    }
  }

  return null;
}

function deduplicateCitations(citations) {
  const seen = new Set();
  const unique = [];

  for (const citation of citations) {
    const key = `${citation.authors}_${citation.year}_${citation.title || ''}`;

    if (!seen.has(key)) {
      seen.add(key);
      unique.push(citation);
    }
  }

  return unique;
}

function enrichCitationsWithDOIs(citations, dois) {
  for (const citation of citations) {
    const matchingDOI = dois.find(doi =>
      citation.rawEntry?.includes(doi)
    );

    if (matchingDOI) {
      citation.doi = matchingDOI;
      // Could enrich further with CrossRef API lookup
    }
  }
}

import { PDFDocument } from 'pdf-lib';

async function extractMetadata(pdfBuffer) {
  try {
    const pdfDoc = await PDFDocument.load(pdfBuffer);

    return {
      title: pdfDoc.getTitle() || '',
      author: pdfDoc.getAuthor() || '',
      subject: pdfDoc.getSubject() || '',
      keywords: pdfDoc.getKeywords() || '',
      creator: pdfDoc.getCreator() || '',
      producer: pdfDoc.getProducer() || '',
      creationDate: pdfDoc.getCreationDate() || null,
      modificationDate: pdfDoc.getModificationDate() || null,
      pageCount: pdfDoc.getPageCount()
    };
  } catch (error) {
    console.error('Metadata extraction error:', error.message);
    return {};
  }
}

async function extractTablesFromPDF(buffer) {
  const { text } = await extractText(buffer);
  const tables = [];
  const lines = text.split('\n');

  let inTable = false;
  let currentTable = [];

  for (const line of lines) {
    // Detect table rows (multiple whitespace-separated values)
    const cells = line.split(/\s{2,}/).filter(c => c.trim());

    if (cells.length >= 3) {
      inTable = true;
      currentTable.push(cells);
    } else if (inTable && currentTable.length > 0) {
      // End of table
      tables.push({
        rows: currentTable,
        columns: currentTable[0].length,
        rowCount: currentTable.length
      });
      currentTable = [];
      inTable = false;
    }
  }

  return tables;
}

// extraction-pipeline.js
import fs from 'fs/promises';
import pdfParse from 'pdf-parse';
import Tesseract from 'tesseract.js';
import { PDFDocument } from 'pdf-lib';

class ExtractionPipeline {
  constructor(options = {}) {
    this.useOCR = options.useOCR || false;
    this.extractTables = options.extractTables || false;
    this.extractCitations = options.extractCitations || true;
  }

  /**
   * Extract all data from a PDF
   */
  async extractFromPDF(pdfPath) {
    const buffer = await fs.readFile(pdfPath);

    // Extract basic text
    const textData = await this.extractText(buffer);

    // Try OCR if text extraction yielded little content
    if (this.useOCR && textData.text.length < 500) {
      textData.text = await this.performOCR(pdfPath);
      textData.ocrUsed = true;
    }

    // Extract citations
    const citations = this.extractCitations
      ? await this.extractCitationsFromText(textData.text)
      : [];

    // Extract tables if requested
    const tables = this.extractTables
      ? await this.extractTablesFromPDF(buffer)
      : [];

    // Extract metadata
    const metadata = await this.extractMetadata(buffer);

    return {
      filepath: pdfPath,
      text: textData.text,
      pages: textData.numpages,
      metadata,
      citations,
      tables,
      ocrUsed: textData.ocrUsed || false,
      extractionDate: new Date().toISOString()
    };
  }

  // ... (all methods from previous sections)
}

export default ExtractionPipeline;

function validateExtraction(data) {
  const checks = {
    textLength: data.text.length > 1000,
    hasMetadata: Object.keys(data.metadata).length > 0,
    hasCitations: data.citations.length > 0,
    noGarbledText: !data.text.includes('���'),
    reasonablePageCount: data.pages > 0 && data.pages < 1000
  };

  const issues = [];

  if (!checks.textLength) {
    issues.push('Suspiciously short text - possible extraction failure');
  }

  if (data.ocrUsed && !checks.noGarbledText) {
    issues.push('OCR quality issues detected - contains garbled characters');
  }

  if (!checks.hasCitations) {
    issues.push('No citations found - verify reference section parsing');
  }

  return {
    valid: issues.length === 0,
    issues,
    quality: data.ocrUsed ? 'medium' : 'high',
    checks
  };
}