const fs = require('fs');
const path = require('path');
const { initDatabase } = require('../db');

const sitesDir = path.join(__dirname, '../sites');
const legacySites = [
  { name: 'Top10OnlineSlots', url: 'https://m.top10onlineslots.co.uk/onlineslots/' },
  { name: 'UBetting Best Slots', url: 'https://ubet.co.uk/casino/best-slots-chart' }
];

let sites = [];

try {
  const files = fs.readdirSync(sitesDir).filter(f => f.endsWith('.json'));
  for (const file of files) {
    try {
      const config = JSON.parse(fs.readFileSync(path.join(sitesDir, file), 'utf8'));
      if (config.url && config.name) sites.push(config);
      else console.warn('Skipped', file, '- missing url or name');
    } catch (e) {
      console.warn('Failed to parse', file + ':', e.message);
    }
  }
} catch (e) {
  console.error('Could not read sites directory:', e.message);
}

if (sites.length === 0) {
  console.log('No site configs found, falling back to legacy list');
  sites = legacySites;
} else {
  console.log(`Loaded ${sites.length} site configs from src/sites/`);
}

async function run() {
  console.log('Starting scheduled crawl...');
  await initDatabase();

  const CasinoCrawlerClass = require('../services/crawler');
  const crawler = new CasinoCrawlerClass();

  try {
    await crawler.initialize();

    for (const site of sites) {
      console.log('\n--- Crawling ' + site.name + ' ---');
      const result = await crawler.crawl(site.url, site.name);

      if (result) {
        console.log('Success: found', result.count, 'casinos from', site.name);
      } else {
        console.log('Failed to crawl:', site.name);
      }

      await new Promise(r => setTimeout(r, 5000));
    }
  } catch (error) {
    console.error('Crawl job failed:', error);
  } finally {
    await crawler.close();
    console.log('\nScheduled crawl complete.');
  }
}

if (require.main === module) {
  run();
}

module.exports = { run, sites };