Intial Version

backend/src/scrapers/base-scraper.ts

@@ -0,0 +1,92 @@
+import { Browser, Page, chromium } from 'playwright';
+import { VendorListing, ScrapeResult, ScraperConfig } from './types';
+
+export abstract class BaseScraper {
+  protected config: ScraperConfig;
+  protected browser: Browser | null = null;
+
+  constructor(config: Partial<ScraperConfig> = {}) {
+    this.config = {
+      maxRetries: 3,
+      retryDelay: 2000,
+      timeout: 30000,
+      headless: true,
+      ...config,
+    };
+  }
+
+  abstract getVendorName(): 'eldorado' | 'playerauctions';
+  abstract getTargetUrl(): string;
+  abstract extractListings(page: Page): Promise<VendorListing[]>;
+
+  async scrape(): Promise<ScrapeResult> {
+    const vendor = this.getVendorName();
+    let lastError: string | undefined;
+
+    for (let attempt = 1; attempt <= this.config.maxRetries; attempt++) {
+      try {
+        const listings = await this.performScrape();
+        
+        return {
+          success: true,
+          vendor,
+          listings,
+          scrapedAt: new Date(),
+        };
+      } catch (error) {
+        lastError = error instanceof Error ? error.message : String(error);
+        
+        if (attempt < this.config.maxRetries) {
+          await this.delay(this.config.retryDelay);
+        }
+      }
+    }
+
+    return {
+      success: false,
+      vendor,
+      listings: [],
+      error: lastError,
+      scrapedAt: new Date(),
+    };
+  }
+
+  private async performScrape(): Promise<VendorListing[]> {
+    this.browser = await chromium.launch({ headless: this.config.headless });
+    
+    try {
+      const context = await this.browser.newContext({
+        userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
+      });
+      
+      const page = await context.newPage();
+      page.setDefaultTimeout(this.config.timeout);
+      
+      await page.goto(this.getTargetUrl(), { waitUntil: 'networkidle' });
+      
+      const listings = await this.extractListings(page);
+      
+      await context.close();
+      
+      return listings;
+    } finally {
+      await this.browser?.close();
+      this.browser = null;
+    }
+  }
+
+  protected calculatePricePerMillion(amountAUEC: number, priceUSD: number): number {
+    return (priceUSD / amountAUEC) * 1_000_000;
+  }
+
+  protected delay(ms: number): Promise<void> {
+    return new Promise(resolve => setTimeout(resolve, ms));
+  }
+
+  async close(): Promise<void> {
+    if (this.browser) {
+      await this.browser.close();
+      this.browser = null;
+    }
+  }
+}

backend/src/scrapers/eldorado-scraper.ts

@@ -0,0 +1,151 @@
+import { Page } from 'playwright';
+import { BaseScraper } from './base-scraper';
+import { VendorListing } from './types';
+
+export class EldoradoScraper extends BaseScraper {
+  getVendorName(): 'eldorado' {
+    return 'eldorado';
+  }
+
+  getTargetUrl(): string {
+    return 'https://www.eldorado.gg/star-citizen-auec/g/141-0-0';
+  }
+
+  async extractListings(page: Page): Promise<VendorListing[]> {
+    // Wait for page readiness
+    await page.waitForSelector('text=Star Citizen aUEC', { timeout: 15000 }).catch(() => {});
+    
+    // Wait for price elements to appear
+    await page.waitForTimeout(3000);
+
+    const listings = await page.evaluate(() => {
+      const results: Array<{
+        amountAUEC: number;
+        priceUSD: number;
+        pricePerMillion: number;
+        seller?: string;
+        deliveryTime?: string;
+      }> = [];
+
+      const bodyText = document.body.innerText;
+      const lines = bodyText.split('\n').map(l => l.trim()).filter(l => l.length > 0);
+
+      // Track seen combinations to avoid duplicates
+      const seenListings = new Set<string>();
+
+      for (let i = 0; i < lines.length; i++) {
+        const line = lines[i];
+
+        // Look for "$/M" pattern - this is the direct price per million
+        // Examples: "$0.00007 / M", "$0.00018 / M", "$0.00007/M"
+        const pricePerMMatch = line.match(/\$\s*([\d.]+)\s*\/?\s*\/\s*M/i) || line.match(/\$\s*([\d.]+)\s*\/\s*M/i);
+
+        if (pricePerMMatch) {
+          const pricePerMillion = parseFloat(pricePerMMatch[1]);
+          
+
+
+          // Look for "Min. qty" or "Min qty" nearby to get the quantity
+          let minQtyM = 10000; // Default to 10000M
+          for (let j = Math.max(0, i - 5); j < Math.min(lines.length, i + 5); j++) {
+            const qtyLine = lines[j];
+            // Match patterns like "Min. qty. 6000 M" or "Min qty: 16,000 M"
+            const qtyMatch = qtyLine.match(/Min\.?\s*qty\.?\s*:?\s*([\d,]+)\s*M/i);
+            if (qtyMatch) {
+              minQtyM = parseFloat(qtyMatch[1].replace(/,/g, ''));
+              break;
+            }
+          }
+
+          const amountAUEC = minQtyM * 1_000_000;
+          const priceUSD = pricePerMillion * minQtyM;
+
+          // Find seller name - look both backwards and forwards
+          // For featured seller, name appears BEFORE the price
+          // For other sellers, name appears in a structured list
+          let seller: string | undefined;
+          
+          // Search backwards first (for featured seller and some list items)
+          for (let j = Math.max(0, i - 20); j < i; j++) {
+            const sellerLine = lines[j];
+            
+            // Skip common non-seller text
+            if (
+              sellerLine.includes('$') ||
+              sellerLine.includes('Price') ||
+              sellerLine.includes('qty') ||
+              sellerLine.includes('stock') ||
+              sellerLine.includes('Delivery') ||
+              sellerLine.toLowerCase().includes('review') ||
+              sellerLine.includes('Rating') ||
+              sellerLine.includes('Offer') ||
+              sellerLine.includes('Details') ||
+              sellerLine.includes('FEATURED') ||
+              sellerLine.includes('Other') ||
+              sellerLine.includes('sellers') ||
+              sellerLine.includes('aUEC') ||
+              sellerLine === 'Star Citizen' || // Exclude the game title but not seller names
+              sellerLine.includes('IMF') ||
+              sellerLine.includes('in-game') ||
+              sellerLine.includes('currency') ||
+              sellerLine.length < 3 ||
+              sellerLine.length > 30
+            ) {
+              continue;
+            }
+            
+            // Match seller name patterns - alphanumeric with underscores/hyphens
+            // Allow some special cases like "StarCitizen"
+            if (/^[a-zA-Z][a-zA-Z0-9_-]*$/.test(sellerLine)) {
+              seller = sellerLine;
+              // Don't break - keep looking for a closer match
+            }
+          }
+
+          // Find delivery time
+          let deliveryTime: string | undefined;
+          for (let j = Math.max(0, i - 5); j < Math.min(lines.length, i + 5); j++) {
+            const deliveryLine = lines[j];
+            if (
+              deliveryLine.match(/\d+\s*min/i) ||
+              deliveryLine.match(/\d+\s*hour/i) ||
+              deliveryLine.toLowerCase().includes('instant')
+            ) {
+              deliveryTime = deliveryLine;
+              break;
+            }
+          }
+
+          // Create unique key to avoid duplicates
+          const key = `${pricePerMillion}-${minQtyM}`;
+          if (seenListings.has(key)) continue;
+          seenListings.add(key);
+
+          results.push({
+            amountAUEC,
+            priceUSD,
+            pricePerMillion,
+            seller,
+            deliveryTime,
+          });
+        }
+      }
+
+      return results;
+    });
+
+    const scrapedAt = new Date();
+    const url = this.getTargetUrl();
+
+    return listings.map(listing => ({
+      vendor: 'eldorado' as const,
+      amountAUEC: listing.amountAUEC,
+      priceUSD: listing.priceUSD,
+      pricePerMillion: listing.pricePerMillion,
+      seller: listing.seller,
+      deliveryTime: listing.deliveryTime,
+      scrapedAt,
+      url,
+    }));
+  }
+}

backend/src/scrapers/index.ts

@@ -0,0 +1,6 @@
+export { BaseScraper } from './base-scraper';
+export { EldoradoScraper } from './eldorado-scraper';
+export { PlayerAuctionsScraper } from './playerauctions-scraper';
+export { ScraperService } from './scraper-service';
+export { ScraperScheduler } from './scheduler';
+export type { VendorListing, ScrapeResult, ScraperConfig } from './types';

backend/src/scrapers/manual-scrape.ts

@@ -0,0 +1,49 @@
+import { ScraperService } from './scraper-service';
+
+async function main() {
+  const scraperService = new ScraperService();
+  
+  try {
+    console.log('Scraping in progress...\n');
+    
+    const results = await scraperService.scrapeAll();
+    
+    // Show completion status for each vendor
+    results.forEach(result => {
+      if (result.success) {
+        console.log(`✓ ${result.vendor.charAt(0).toUpperCase() + result.vendor.slice(1)} scraping done`);
+      } else {
+        console.log(`✗ ${result.vendor.charAt(0).toUpperCase() + result.vendor.slice(1)} scraping failed`);
+      }
+    });
+    
+    // Show listings from each vendor
+    console.log('');
+    results.forEach(result => {
+      console.log(`[${result.vendor.toUpperCase()}] Found ${result.listings.length} listings`);
+      if (result.listings.length > 0) {
+        result.listings.forEach((listing, i) => {
+          console.log(`  ${i + 1}. $${listing.pricePerMillion}/M (${listing.seller || 'Unknown'})`);
+        });
+      }
+      console.log('');
+    });
+    
+    const allListings = results.flatMap(r => r.listings);
+    const lowestPrice = scraperService.calculatePriceIndex(allListings);
+    
+    console.log('=== LOWEST PRICE ===');
+    if (lowestPrice) {
+      console.log(`$${lowestPrice}/M`);
+    } else {
+      console.log('No listings found');
+    }
+  } catch (error) {
+    console.error('Error:', error);
+    process.exit(1);
+  } finally {
+    await scraperService.close();
+  }
+}
+
+main();

backend/src/scrapers/playerauctions-scraper.ts

@@ -0,0 +1,202 @@
+import { Page } from 'playwright';
+import { BaseScraper } from './base-scraper';
+import { VendorListing } from './types';
+
+export class PlayerAuctionsScraper extends BaseScraper {
+  getVendorName(): 'playerauctions' {
+    return 'playerauctions';
+  }
+
+  getTargetUrl(): string {
+    return 'https://www.playerauctions.com/star-citizen-auec/';
+  }
+
+  async extractListings(page: Page): Promise<VendorListing[]> {
+    // Wait for page readiness
+    await page.waitForTimeout(3000);
+
+    // Close cookie popup if it exists
+    try {
+      const cookieClose = page.locator('[id*="cookie"] button, [class*="cookie"] button, button:has-text("Accept"), button:has-text("Close")').first();
+      if (await cookieClose.isVisible({ timeout: 2000 }).catch(() => false)) {
+        await cookieClose.click();
+        await page.waitForTimeout(500);
+      }
+    } catch (e) {
+      // No cookie popup or already closed
+    }
+
+    // Find offer cards - they have class "offer-item"
+    const offerCards = await page.locator('.offer-item, [class*="offer-item"]').all();
+
+    if (offerCards.length === 0) {
+      return this.extractListingsAlternative(page);
+    }
+
+    const listings: VendorListing[] = [];
+    const targetQuantityM = 100000; // 10000 M = 10 billion AUEC (field is already in millions)
+
+    // Step 2-5: Process each offer card
+    for (let i = 0; i < Math.min(offerCards.length, 20); i++) {
+      try {
+        const card = offerCards[i];
+
+        // Find the quantity input (shows number with "M" suffix, has +/- buttons)
+        const qtyInput = card.locator('input[type="number"]').first();
+        
+        if (!(await qtyInput.isVisible({ timeout: 1000 }).catch(() => false))) {
+          continue;
+        }
+
+        // Set quantity to 10000 (which means 10000 M = 10 billion AUEC)
+        await qtyInput.scrollIntoViewIfNeeded();
+        await qtyInput.click({ force: true });
+        await qtyInput.fill('');
+        await qtyInput.pressSequentially(targetQuantityM.toString(), { delay: 10 });
+        await qtyInput.press('Enter'); // Trigger update
+        
+        // Wait for price to update (0.5-2 seconds as per instructions)
+        await page.waitForTimeout(2500);
+
+        // Step 3: Extract the total price from the BUY NOW button area
+        // Look for the price near the BUY NOW button - it's typically in a large font
+        let totalPriceUSD = 0;
+        
+        // Try to find the price element near BUY NOW button
+        const buyNowButton = card.locator('button:has-text("BUY NOW"), [class*="buy"]').first();
+        if (await buyNowButton.isVisible().catch(() => false)) {
+          // Get the parent container and look for price nearby
+          const priceContainer = buyNowButton.locator('xpath=..').first();
+          const priceText = await priceContainer.textContent().catch(() => '');
+          
+          // Extract price - should be like "$5.00" in large text
+          if (priceText) {
+            const priceMatch = priceText.match(/\$\s*([\d,]+\.\d{2})/);
+            if (priceMatch) {
+              totalPriceUSD = parseFloat(priceMatch[1].replace(/,/g, ''));
+            }
+          }
+        }
+        
+        // Fallback: look for price in the card, but exclude "Minutes" context
+        if (totalPriceUSD === 0) {
+          const cardText = await card.textContent().catch(() => '');
+          if (cardText) {
+            const lines = cardText.split('\n').map(l => l.trim());
+          
+          for (const line of lines) {
+            // Skip lines that contain time indicators
+            if (line.includes('Minutes') || line.includes('Hours') || line.includes('Days')) {
+              continue;
+            }
+            
+            // Look for price pattern with decimal
+            const priceMatch = line.match(/\$\s*([\d,]+\.\d{2})/);
+            if (priceMatch) {
+              const price = parseFloat(priceMatch[1].replace(/,/g, ''));
+              if (price > 0 && price < 100000) {
+                totalPriceUSD = price;
+                break;
+              }
+            }
+          }
+          }
+        }
+        
+        if (totalPriceUSD === 0) {
+          continue;
+        }
+
+        // Step 4: Compute USD per 1M
+        const pricePerMillion = totalPriceUSD / targetQuantityM;
+
+        // Extract seller name and delivery time from card text
+        const fullCardText = await card.textContent().catch(() => '');
+        const sellerMatch = fullCardText ? fullCardText.match(/([a-zA-Z0-9_-]{3,20})/) : null;
+        const seller = sellerMatch ? sellerMatch[1] : 'Unknown';
+
+        const deliveryMatch = fullCardText ? fullCardText.match(/(\d+\s*(?:Minutes?|Hours?|Days?))/i) : null;
+        const deliveryTime = deliveryMatch ? deliveryMatch[1] : undefined;
+
+        listings.push({
+          vendor: 'playerauctions',
+          amountAUEC: targetQuantityM * 1_000_000,
+          priceUSD: totalPriceUSD,
+          pricePerMillion,
+          seller: seller.trim(),
+          deliveryTime,
+          scrapedAt: new Date(),
+          url: this.getTargetUrl(),
+        });
+
+      } catch (error) {
+        // Skip this card
+      }
+    }
+
+    if (listings.length === 0) {
+      return this.extractListingsAlternative(page);
+    }
+
+    return listings;
+  }
+
+  private async extractListingsAlternative(page: Page): Promise<VendorListing[]> {
+
+    const listings = await page.evaluate(() => {
+      const results: Array<{
+        amountAUEC: number;
+        priceUSD: number;
+        pricePerMillion: number;
+        seller?: string;
+      }> = [];
+
+      const bodyText = document.body.innerText;
+      const lines = bodyText.split('\n').map(l => l.trim()).filter(l => l.length > 0);
+
+      const seenPrices = new Set<number>();
+
+      for (let i = 0; i < lines.length; i++) {
+        const line = lines[i];
+
+        // Look for "$1 = X M aUEC" pattern and convert
+        const exchangeMatch = line.match(/\$1\s*=\s*([\d,]+(?:\.\d+)?)\s*M\s*(?:aUEC)?/i);
+        
+        if (exchangeMatch) {
+          const millionsPerDollar = parseFloat(exchangeMatch[1].replace(/,/g, ''));
+          const pricePerMillion = 1 / millionsPerDollar;
+
+          if (seenPrices.has(pricePerMillion)) continue;
+          seenPrices.add(pricePerMillion);
+
+          const targetQuantityM = 100000;
+          const amountAUEC = targetQuantityM * 1_000_000;
+          const priceUSD = pricePerMillion * targetQuantityM;
+
+          results.push({
+            amountAUEC,
+            priceUSD,
+            pricePerMillion,
+            seller: 'Unknown',
+          });
+        }
+      }
+
+      return results;
+    });
+
+    const scrapedAt = new Date();
+    const url = this.getTargetUrl();
+
+    return listings.map(listing => ({
+      vendor: 'playerauctions' as const,
+      amountAUEC: listing.amountAUEC,
+      priceUSD: listing.priceUSD,
+      pricePerMillion: listing.pricePerMillion,
+      seller: listing.seller,
+      deliveryTime: undefined,
+      scrapedAt,
+      url,
+    }));
+  }
+}

backend/src/scrapers/scheduler.ts

@@ -0,0 +1,80 @@
+import * as schedule from 'node-schedule';
+import { ScraperService } from './scraper-service';
+import { ScrapeResult } from './types';
+
+export type ScrapeCallback = (results: ScrapeResult[]) => Promise<void>;
+
+export class ScraperScheduler {
+  private scraperService: ScraperService;
+  private job: schedule.Job | null = null;
+  private callback: ScrapeCallback | null = null;
+  private intervalMinutes: number;
+
+  constructor(intervalMinutes: number = 5) {
+    this.scraperService = new ScraperService();
+    this.intervalMinutes = intervalMinutes;
+  }
+
+  onScrapeComplete(callback: ScrapeCallback): void {
+    this.callback = callback;
+  }
+
+  start(): void {
+    if (this.job) {
+      console.log('Scheduler already running');
+      return;
+    }
+
+    // Run immediately on start
+    this.runScrape();
+
+    // Schedule recurring scrapes
+    const rule = `*/${this.intervalMinutes} * * * *`;
+    this.job = schedule.scheduleJob(rule, () => {
+      this.runScrape();
+    });
+
+    console.log(`Scraper scheduled to run every ${this.intervalMinutes} minutes`);
+  }
+
+  stop(): void {
+    if (this.job) {
+      this.job.cancel();
+      this.job = null;
+      console.log('Scraper scheduler stopped');
+    }
+  }
+
+  async runScrape(): Promise<void> {
+    try {
+      console.log(`[${new Date().toISOString()}] Running scheduled scrape...`);
+      
+      const results = await this.scraperService.scrapeAll();
+      
+      const successCount = results.filter(r => r.success).length;
+      const totalListings = results.reduce((sum, r) => sum + r.listings.length, 0);
+      
+      console.log(`Scrape complete: ${successCount}/${results.length} vendors successful, ${totalListings} listings`);
+      
+      if (this.callback) {
+        await this.callback(results);
+      }
+    } catch (error) {
+      console.error('Error during scheduled scrape:', error);
+    }
+  }
+
+  setInterval(minutes: number): void {
+    this.intervalMinutes = minutes;
+    
+    if (this.job) {
+      this.stop();
+      this.start();
+    }
+  }
+
+  async close(): Promise<void> {
+    this.stop();
+    await this.scraperService.close();
+  }
+}

backend/src/scrapers/scraper-service.ts

@@ -0,0 +1,59 @@
+import { EldoradoScraper } from './eldorado-scraper';
+import { PlayerAuctionsScraper } from './playerauctions-scraper';
+import { ScrapeResult, VendorListing } from './types';
+
+export class ScraperService {
+  private eldoradoScraper: EldoradoScraper;
+  private playerAuctionsScraper: PlayerAuctionsScraper;
+
+  constructor() {
+    this.eldoradoScraper = new EldoradoScraper();
+    this.playerAuctionsScraper = new PlayerAuctionsScraper();
+  }
+
+  async scrapeAll(): Promise<ScrapeResult[]> {
+    const results = await Promise.allSettled([
+      this.eldoradoScraper.scrape(),
+      this.playerAuctionsScraper.scrape(),
+    ]);
+
+    return results.map((result, index) => {
+      if (result.status === 'fulfilled') {
+        return result.value;
+      } else {
+        const vendor = index === 0 ? 'eldorado' : 'playerauctions';
+        return {
+          success: false,
+          vendor,
+          listings: [],
+          error: result.reason?.message || 'Unknown error',
+          scrapedAt: new Date(),
+        };
+      }
+    });
+  }
+
+  async scrapeEldorado(): Promise<ScrapeResult> {
+    return this.eldoradoScraper.scrape();
+  }
+
+  async scrapePlayerAuctions(): Promise<ScrapeResult> {
+    return this.playerAuctionsScraper.scrape();
+  }
+
+  calculatePriceIndex(listings: VendorListing[]): number | null {
+    if (listings.length === 0) return null;
+
+    const prices = listings.map(l => l.pricePerMillion);
+    
+    // Return the lowest price
+    return Math.min(...prices);
+  }
+
+  async close(): Promise<void> {
+    await Promise.all([
+      this.eldoradoScraper.close(),
+      this.playerAuctionsScraper.close(),
+    ]);
+  }
+}

backend/src/scrapers/types.ts

@@ -0,0 +1,25 @@
+export interface VendorListing {
+  vendor: 'eldorado' | 'playerauctions';
+  amountAUEC: number;
+  priceUSD: number;
+  pricePerMillion: number;
+  seller?: string;
+  deliveryTime?: string;
+  scrapedAt: Date;
+  url: string;
+}
+
+export interface ScrapeResult {
+  success: boolean;
+  vendor: string;
+  listings: VendorListing[];
+  error?: string;
+  scrapedAt: Date;
+}
+
+export interface ScraperConfig {
+  maxRetries: number;
+  retryDelay: number;
+  timeout: number;
+  headless: boolean;
+}