Football World Cup 2026

import Firecrawl from "@mendable/firecrawl-js";
import * as cheerio from "cheerio";

const apiKey = process.env.FIRECRAWL_API_KEY;
if (!apiKey) {
  console.error("FIRECRAWL_API_KEY is not set");
  process.exit(1);
}
const firecrawl = new Firecrawl({ apiKey });

// Polymarket Gamma API tag id for "FIFA World Cup" (the 2026 tournament).
const TAG_ID = 102232;
// Pages of 100 events grew past Firecrawl's file size limit (~40MB of JSON);
// 20 events per page stays well under it, and fetchRange splits further on demand.
const PAGE_SIZE = 20;
const MAX_PAGES = 100;
// Firecrawl occasionally fails with transient service errors ("OAuth
// authentication is temporarily unavailable", rate limits, 5xx); retry those.
const MAX_RETRIES = 5;

function isSizeLimitError(err: unknown): boolean {
  return /exceeds size limit|file type that Firecrawl cannot process/i.test(String(err));
}

function isTransientError(err: unknown): boolean {
  const s = String(err);
  return /OAuth authentication is temporarily unavailable|temporarily unavailable|rate limit|too many requests|429|50[234]|timed? ?out|ECONNRESET|socket hang up|network/i.test(
    s,
  );
}

function sleep(ms: number): Promise<void> {
  return new Promise((resolve) => setTimeout(resolve, ms));
}

async function scrapeWithRetry(url: string): Promise<any> {
  let lastErr: unknown;
  for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) {
    try {
      return await firecrawl.scrape(url, { formats: ["rawHtml"], integration: "prometheus" });
    } catch (err) {
      if (isSizeLimitError(err) || !isTransientError(err)) throw err;
      lastErr = err;
      const delay = 2000 * attempt;
      console.error(`Transient Firecrawl error (attempt ${attempt}/${MAX_RETRIES}): ${String(err)}; retrying in ${delay}ms...`);
      await sleep(delay);
    }
  }
  throw new Error(`Firecrawl request failed after ${MAX_RETRIES} retries: ${String(lastErr)}`);
}

async function fetchRange(offset: number, count: number): Promise<any[]> {
  const url = `https://gamma-api.polymarket.com/events?tag_id=${TAG_ID}&closed=false&limit=${count}&offset=${offset}`;
  let res: any;
  try {
    res = await scrapeWithRetry(url);
  } catch (err) {
    // The JSON for this slice is too large for Firecrawl; split it in half.
    if (isSizeLimitError(err) && count > 1) {
      const half = Math.ceil(count / 2);
      console.error(`Slice at offset ${offset} (limit ${count}) too large; splitting into ${half}-event slices...`);
      const first = await fetchRange(offset, half);
      if (first.length < half) return first;
      const second = await fetchRange(offset + half, count - half);
      return first.concat(second);
    }
    if (isSizeLimitError(err)) {
      throw new Error(`Gamma API single event at offset ${offset} exceeds Firecrawl size limit`);
    }
    throw err;
  }
  let raw: string = res?.rawHtml ?? "";
  if (raw.trimStart().startsWith("<")) {
    // Some renderers wrap JSON responses in an HTML page; recover the text.
    raw = cheerio.load(raw)("pre").text() || cheerio.load(raw)("body").text();
  }
  let parsed: any;
  try {
    parsed = JSON.parse(raw);
  } catch {
    throw new Error(`Gamma API response at offset ${offset} was not valid JSON`);
  }
  if (!Array.isArray(parsed)) {
    throw new Error(`Gamma API response at offset ${offset} was not an array of events`);
  }
  return parsed;
}

function parseJsonArray(s: unknown): any[] {
  if (typeof s !== "string") return [];
  try {
    const v = JSON.parse(s);
    return Array.isArray(v) ? v : [];
  } catch {
    return [];
  }
}

function toNumber(v: unknown): number | null {
  if (v === null || v === undefined || v === "") return null;
  const n = Number(v);
  return Number.isFinite(n) ? n : null;
}

function mapMarket(m: any) {
  const outcomeNames = parseJsonArray(m.outcomes);
  const outcomePrices = parseJsonArray(m.outcomePrices);
  const outcomes = outcomeNames.map((name: string, i: number) => ({
    name,
    price: toNumber(outcomePrices[i]),
  }));
  return {
    id: m.id ?? null,
    question: m.question ?? null,
    groupItemTitle: m.groupItemTitle || null,
    slug: m.slug ?? null,
    outcomes,
    volumeUsd: toNumber(m.volume),
    liquidityUsd: toNumber(m.liquidity),
    endDate: m.endDate ?? null,
    active: m.active ?? null,
    closed: m.closed ?? null,
  };
}

function mapEvent(e: any) {
  const title: string = e.title ?? "";
  const slug: string = e.slug ?? "";
  const isMatch = / vs\.? /i.test(title) || slug.startsWith("fifwc-");
  return {
    id: e.id ?? null,
    title: title.trim(),
    slug,
    url: slug ? `https://polymarket.com/event/${slug}` : null,
    category: isMatch ? "match" : "futures-and-props",
    description: e.description ?? null,
    startDate: e.startDate ?? null,
    endDate: e.endDate ?? null,
    volumeUsd: toNumber(e.volume),
    liquidityUsd: toNumber(e.liquidity),
    commentCount: e.commentCount ?? null,
    markets: Array.isArray(e.markets) ? e.markets.map(mapMarket) : [],
  };
}

async function main() {
  const events: any[] = [];
  const seen = new Set<string>();
  for (let page = 0; page < MAX_PAGES; page++) {
    const offset = page * PAGE_SIZE;
    console.error(`Fetching events at offset ${offset}...`);
    const batch = await fetchRange(offset, PAGE_SIZE);
    for (const e of batch) {
      const id = String(e.id ?? e.slug ?? "");
      if (id && seen.has(id)) continue;
      if (id) seen.add(id);
      events.push(mapEvent(e));
    }
    if (batch.length < PAGE_SIZE) break;
  }
  if (events.length === 0) {
    throw new Error("no open FIFA World Cup events returned by the Gamma API");
  }
  events.sort((a, b) => (b.volumeUsd ?? 0) - (a.volumeUsd ?? 0));
  const marketCount = events.reduce((n, e) => n + e.markets.length, 0);
  console.error(`Collected ${events.length} events with ${marketCount} markets.`);
  const out = {
    tournament: "FIFA World Cup 2026",
    source: "Polymarket (Gamma API)",
    eventCount: events.length,
    marketCount,
    events,
  };
  process.stdout.write(JSON.stringify(out));
}

main().catch((err) => {
  console.error(err);
  process.exit(1);
});