derma/bin/scrape.mjs

import { get_page_events } from '../src/facebook/get-page-events.mjs';
import { get_credentials, unix, updated } from '../src/util.mjs';
import { update_last_scraped } from '../src/api/places.mjs';
import send from '../src/signal/send.mjs';
import fetch from 'node-fetch';

const prod = true;
const { api, token } = get_credentials(prod);

const headers = { 'Content-Type': 'application/json' };

const scrape = async (pageID) => {
  try {
    const res = await get_page_events({
      pageID,
      get_upcoming_events: true,
      get_past_events: false
    });
    return res;
  } catch (e) {
    console.error(e);
  }
  return [];
};

(async () => {
  let resp = await fetch(`${api}/places/?token=${token}`);
  let places = await resp.json();
  places = places.filter((place) => {
    const scrape = place.scraper == 'facebook';
    if (!scrape) {
      console.log(
        101,
        `Skipping #${place.id} ${place.name}. Reason: Scraper is ${place.scraper}`
      );
      return false;
    }
    const now = unix(new Date());
    const recently = place.last_scraped + place.scrape_threshold;
    if (now < recently) {
      console.log(
        100,
        `Skipping #${place.id} ${place.name}. Reason: Was scraped ${
          now - place.last_scraped
        }s ago.`
      );
      return false;
    }
    return true;
  });

  let skuret = places.findIndex((place) => place.id == 50);
  places = places.filter((place) => place.id != 50);
  places = [...places, skuret];

  for (let place of places) {
    console.log(
      177,
      `Scraping #${place.id} ${place.name}. {facebook_id: ${place.facebook_id}, facebook_name_id: "${place.facebook_name_id}"}`
    );
    const events = await scrape(place.facebook_id);
    let payloads = [];
    for (let event of events) {
      payloads.push({
        canceled: false,
        end: -1,
        start: unix(new Date(event.time_range.start)),
        draft: false,
        facebook_id: event.id,
        place_id: place.id,
        name: event.name ?? '',
        ticket_url: event.event_buy_ticket_url ?? ''
      });
    }
    if (payloads.length == 0) {
      console.log(123, 'No upcoming events, dead place?');
    }
    for (let payload of payloads) {
      let search = await fetch(
        `${api}/search/events/?facebook_id=${payload.facebook_id}&token=${token}`
      );
      if (!search.ok) {
        console.log(500, await search.text());
        continue;
      }
      search = await search.json();
      let new_event = search.length === 0;
      let old_event;
      if (!new_event) {
        old_event = search[0];
      }
      let res;
      if (new_event) {
        res = await fetch(`${api}/events/?token=${token}`, {
          method: 'POST',
          body: JSON.stringify(payload),
          headers
        });
        console.log(res.status, 'Insert', place.name, payload.name);
        let newEvent = await res.text();
        let msg = await send(newEvent, place);
        console.log(res.status, 'Signal', msg);
      } else if (old_event && updated(old_event, payload)) {
        payload.id = old_event.id;
        if (old_event.ticket_url.length > 0 && payload.ticket_url.length == 0) {
          payload.ticket_url = old_event.ticket_url;
        }
        if (updated(old_event, payload)) {
          res = await fetch(`${api}/events/${old_event.id}/?token=${token}`, {
            method: 'PATCH',
            body: JSON.stringify(payload),
            headers
          });
          console.log(res.status, 'Update', place.name, payload.name);
        } else {
          console.log(201, 'Skip Update', place.name, payload.name);
        }
      } else {
        console.log(201, 'Skip', place.name, payload.name);
      }
    }
    if (payloads.length > 0) {
      await update_last_scraped(place, prod);
    }
  }
})();