Gojek Indonesian User Reviews

import Firecrawl from "@mendable/firecrawl-js";
import * as cheerio from "cheerio";

const apiKey = process.env.FIRECRAWL_API_KEY;
if (!apiKey) {
  console.error("FIRECRAWL_API_KEY is not set");
  process.exit(1);
}

const firecrawl = new Firecrawl({ apiKey });
const TARGET_COUNT = 200;

type ReviewRow = {
  review_text: string;
  rating_if_available: number | null;
  review_date: string;
  reviewer_name: string;
  page_title: string;
  source_site: string;
  source_url: string;
};

function normalizeText(value: string): string {
  return value.replace(/\s+/g, " ").trim();
}

function dedupeKey(value: string): string {
  return normalizeText(value).toLowerCase();
}

function decodeJsonString(value: string): string {
  try {
    return JSON.parse(`"${value.replace(/\n/g, "\\n")}"`);
  } catch {
    return value;
  }
}

function looksIndonesian(text: string): boolean {
  const lowered = ` ${text.toLowerCase()} `;
  const markers = [
    " yang ",
    " dan ",
    " di ",
    " ke ",
    " dari ",
    " untuk ",
    " dengan ",
    " karena ",
    " kalau ",
    " saya ",
    " aku ",
    " ga ",
    " gak ",
    " nggak ",
    " tidak ",
    " bisa ",
    " banget ",
    " udah ",
    " sudah ",
    " aja ",
    " tapi ",
    " driver ",
    " aplikasi ",
    " pesanan ",
    " makanan ",
    " saldo ",
    " promo ",
    " gofood ",
    " goride ",
    " gopay ",
    " gojek ",
    " mahal ",
    " lama ",
    " susah ",
    " tolong ",
    " kecewa ",
    " membantu ",
  ];
  return markers.some((marker) => lowered.includes(marker));
}

function addReview(rows: ReviewRow[], seen: Set<string>, row: ReviewRow): void {
  row.review_text = normalizeText(row.review_text);
  row.reviewer_name = normalizeText(row.reviewer_name);
  row.page_title = normalizeText(row.page_title);

  if (row.review_text.length < 20) return;
  if (!looksIndonesian(row.review_text)) return;

  const key = dedupeKey(row.review_text);
  if (seen.has(key)) return;

  seen.add(key);
  rows.push(row);
}

async function scrapeRawHtml(url: string): Promise<string> {
  const doc = (await firecrawl.scrape(url, {
    formats: ["rawHtml"],
    integration: "prometheus",
  })) as { rawHtml?: string; html?: string };

  const rawHtml = doc.rawHtml ?? doc.html ?? "";
  if (!rawHtml) {
    throw new Error(`no raw HTML returned for ${url}`);
  }
  return rawHtml;
}

async function collectGooglePlay(rows: ReviewRow[], seen: Set<string>): Promise<void> {
  const sourceUrl = "https://play.google.com/store/apps/details?id=com.gojek.app&hl=id&gl=ID";
  const rawHtml = await scrapeRawHtml(sourceUrl);
  const pageTitle = normalizeText(cheerio.load(rawHtml)("title").first().text()) || "Gojek - Google Play";

  const reviewPattern =
    /\["([a-f0-9-]{36})",\["((?:\\.|[^"\\])*)",[\s\S]{0,1400}?\],(\d),null,"((?:\\.|[^"\\])*)",\[(\d{10}),\d+\]/g;

  let match: RegExpExecArray | null;
  while ((match = reviewPattern.exec(rawHtml)) !== null) {
    const reviewerName = decodeJsonString(match[2]);
    const rating = Number(match[3]);
    const reviewText = decodeJsonString(match[4]);
    const reviewDate = new Date(Number(match[5]) * 1000).toISOString();

    addReview(rows, seen, {
      review_text: reviewText,
      rating_if_available: Number.isFinite(rating) ? rating : null,
      review_date: reviewDate,
      reviewer_name: reviewerName,
      page_title: pageTitle,
      source_site: "Google Play",
      source_url: sourceUrl,
    });
  }
}

async function collectAppleAppStore(rows: ReviewRow[], seen: Set<string>): Promise<void> {
  for (let page = 1; page <= 10 && rows.length < TARGET_COUNT; page += 1) {
    const sourceUrl = `https://itunes.apple.com/id/rss/customerreviews/page=${page}/id=944875099/sortby=mostrecent/xml`;
    const rawXml = await scrapeRawHtml(sourceUrl);
    const $ = cheerio.load(rawXml, { xmlMode: true });

    $("entry").each((_, entry) => {
      if (rows.length >= TARGET_COUNT) return false;

      const reviewText = $(entry).find('content[type="text"]').first().text();
      const ratingText = $(entry).find("im\\:rating").first().text();
      const rating = Number(ratingText);
      const reviewDate = $(entry).find("updated").first().text();
      const reviewerName = $(entry).find("author > name").first().text();

      addReview(rows, seen, {
        review_text: reviewText,
        rating_if_available: Number.isFinite(rating) ? rating : null,
        review_date: reviewDate,
        reviewer_name: reviewerName,
        page_title: "Gojek - Ratings and Reviews - App Store",
        source_site: "Apple App Store",
        source_url: sourceUrl,
      });

      return undefined;
    });
  }
}

async function main() {
  const rows: ReviewRow[] = [];
  const seen = new Set<string>();

  await collectGooglePlay(rows, seen);
  await collectAppleAppStore(rows, seen);

  if (rows.length < 150) {
    throw new Error(`collected fewer review rows than expected: ${rows.length}`);
  }

  process.stdout.write(JSON.stringify(rows.slice(0, TARGET_COUNT)));
}

main().catch((err) => {
  console.error(err);
  process.exit(1);
});