Download Management

import fs from 'fs/promises';
import path from 'path';
import { chromium } from 'playwright';
import pLimit from 'p-limit';

class DownloadManager {
  constructor(options = {}) {
    this.downloadDir = options.downloadDir || './papers';
    this.concurrency = options.concurrency || 5; // Parallel downloads
    this.timeout = options.timeout || 60000; // 60 seconds per download
    this.retryAttempts = options.retryAttempts || 3;
    this.authState = options.authState || null;

    this.stats = {
      total: 0,
      completed: 0,
      failed: 0,
      skipped: 0
    };
  }

  /**
   * Download multiple PDFs in parallel
   */
  async downloadBatch(papers, progressCallback = null) {
    await this.ensureDownloadDir();

    const limit = pLimit(this.concurrency);
    this.stats.total = papers.length;

    const downloads = papers.map(paper =>
      limit(() => this.downloadPaper(paper, progressCallback))
    );

    const results = await Promise.allSettled(downloads);

    return {
      stats: this.stats,
      results: results.map((r, i) => ({
        paper: papers[i],
        status: r.status,
        result: r.value || r.reason
      }))
    };
  }

  /**
   * Download a single paper with retry logic
   */
  async downloadPaper(paper, progressCallback) {
    const filename = this.generateFilename(paper);
    const filepath = path.join(this.downloadDir, filename);

    // Check if already downloaded
    const exists = await this.fileExists(filepath);
    if (exists) {
      const size = await this.getFileSize(filepath);
      if (size > 10000) { // At least 10KB
        this.stats.skipped++;
        if (progressCallback) {
          progressCallback({ paper, status: 'skipped', filepath });
        }
        return { status: 'skipped', filepath };
      }
    }

    // Try downloading with retries
    for (let attempt = 1; attempt <= this.retryAttempts; attempt++) {
      try {
        const result = await this.attemptDownload(paper, filepath);

        this.stats.completed++;
        if (progressCallback) {
          progressCallback({ paper, status: 'completed', filepath, attempt });
        }

        return result;

      } catch (error) {
        console.error(`Download attempt ${attempt} failed for ${paper.title}:`, error.message);

        if (attempt === this.retryAttempts) {
          this.stats.failed++;
          if (progressCallback) {
            progressCallback({ paper, status: 'failed', error: error.message });
          }
          throw error;
        }

        // Exponential backoff
        await this.delay(Math.pow(2, attempt) * 1000);
      }
    }
  }

  /**
   * Attempt to download PDF using Playwright
   */
  async attemptDownload(paper, filepath) {
    if (!paper.pdfUrl) {
      throw new Error('No PDF URL available');
    }

    const browser = await chromium.launch({ headless: true });
    const contextOptions = {
      acceptDownloads: true,
      viewport: { width: 1920, height: 1080 }
    };

    if (this.authState) {
      contextOptions.storageState = this.authState;
    }

    const context = await browser.newContext(contextOptions);
    const page = await context.newPage();

    try {
      // Set download behavior
      await page.setExtraHTTPHeaders({
        'Accept': 'application/pdf,application/octet-stream'
      });

      // Navigate to PDF URL
      const response = await page.goto(paper.pdfUrl, {
        waitUntil: 'networkidle',
        timeout: this.timeout
      });

      // Check if response is PDF
      const contentType = response.headers()['content-type'];

      if (contentType?.includes('application/pdf')) {
        // Direct PDF download
        const buffer = await response.body();
        await fs.writeFile(filepath, buffer);
        await browser.close();
        return { status: 'completed', filepath, size: buffer.length };
      }

      // Handle download button click (indirect PDF)
      const downloadPromise = page.waitForEvent('download', {
        timeout: this.timeout
      });

      // Try common download button patterns
      const downloaded = await this.clickDownloadButton(page);

      if (downloaded) {
        const download = await downloadPromise;
        await download.saveAs(filepath);
        await browser.close();
        return { status: 'completed', filepath };
      }

      throw new Error('Could not trigger PDF download');

    } finally {
      await browser.close();
    }
  }

  /**
   * Try clicking various download button patterns
   */
  async clickDownloadButton(page) {
    const selectors = [
      'a[href$=".pdf"]',
      'button:has-text("Download")',
      'a:has-text("PDF")',
      '.download-pdf',
      '[aria-label*="download"]',
      '[data-test="download-button"]'
    ];

    for (const selector of selectors) {
      try {
        const button = await page.$(selector);
        if (button && await button.isVisible()) {
          await button.click();
          await page.waitForTimeout(1000);
          return true;
        }
      } catch (error) {
        // Try next selector
        continue;
      }
    }

    return false;
  }