derma/bin/scrape.mjs

import { get_page_events } from '../src/facebook/get-page-events.mjs';
import send from '../src/signal/send.mjs';
import fetch from 'node-fetch';

const api = 'http://localhost:3333';
const token =
  //  '831411806230c7e950c4eeb226499ef92bb6bdc4157797929a0e16d133dc13a8';
  '1234567812345678123456781234567812345678123456781234567812345678';
const headers = { 'Content-Type': 'application/json' };

const scrape = async (pageID) => {
  try {
    const res = await get_page_events({
      pageID,
      get_upcoming_events: true,
      get_past_events: false
    });
    return res;
  } catch (e) {
    console.error(e);
  }
  return [];
};
const unix = (a) => parseInt(new Date(a).valueOf() / 1000, 10);
const sleep = (ms) => new Promise((res) => setTimeout(res, ms));
const updated = (oldEvent, scrapedEvent) => {
  let keys = [
    'canceled',
    'end',
    'start',
    'draft',
    'facebook_id',
    'place_id',
    'name',
    'ticket_url'
  ];
  for (let key of keys) {
    if (oldEvent[key] != scrapedEvent[key]) {
      console.log(124, oldEvent[key], '!=', scrapedEvent[key]);
      return true;
    }
  }
  return false;
};

(async () => {
  let resp = await fetch(`${api}/places/?token=${token}`);
  let places = await resp.json();
  places = places.filter((place) => {
    const scrape = place.scraper == 'facebook';
    if (!scrape) {
      console.log(
        101,
        `Skipping #${place.id} ${place.name}. Reason: Scraper is ${place.scraper}`
      );
      return false;
    }
    const now = unix(new Date());
    const recently = place.last_scraped + place.scrape_threshold;
    if (now < recently) {
      console.log(
        100,
        `Skipping #${place.id} ${place.name}. Reason: Was scraped ${
          now - place.last_scraped
        }s ago.`
      );
      return false;
    }

    return true;
  });

  for (let place of places) {
    console.log(177, `Scraping #${place.id} ${place.name}`);
    const events = await scrape(place.facebook_id);
    let payloads = [];
    for (let event of events) {
      payloads.push({
        canceled: false,
        end: -1,
        start: unix(new Date(event.time_range.start)),
        draft: false,
        facebook_id: event.id,
        place_id: place.id,
        name: event.name ?? '',
        ticket_url: event.event_buy_ticket_url ?? ''
      });
    }
    if (payloads.length < 2) {
      console.log(123, JSON.stringify(payloads));
    }
    for (let payload of payloads) {
      let search = await fetch(
        `${api}/search/events/?facebook_id=${payload.facebook_id}&token=${token}`
      );
      if (!search.ok) {
        console.log(500, await search.text());
        continue;
      }
      search = await search.json();
      let new_event = search.length === 0;
      let old_event;
      if (!new_event) {
        old_event = search[0];
      }
      let res;
      if (new_event) {
        res = await fetch(`${api}/events/?token=${token}`, {
          method: 'POST',
          body: JSON.stringify(payload),
          headers
        });
        console.log(res.status, 'Insert', place.name, payload.name);
        let newEvent = await res.text();
        let msg = await send(newEvent, place);
        console.log(res.status, 'Signal', msg);
      } else if (old_event && updated(old_event, payload)) {
        payload.id = old_event.id;
        if (old_event.ticket_url.length > 0 && payload.ticket_url.length == 0) {
          payload.ticket_url = old_event.ticket_url;
        }
        if (updated(old_event, payload)) {
          res = await fetch(`${api}/events/${old_event.id}/?token=${token}`, {
            method: 'PATCH',
            body: JSON.stringify(payload),
            headers
          });
          console.log(res.status, 'Update', place.name, payload.name);
        } else {
          console.log(201, 'Skip Update', place.name, payload.name);
        }
      } else {
        console.log(201, 'Skip', place.name, payload.name);
      }
    }
    let res = await fetch(`${api}/places/${place.id}/?token=${token}`, {
      method: 'PATCH',
      body: JSON.stringify({
        last_scraped: unix(new Date())
      }),
      headers
    });
    console.log(res.status, 'Updated last scraped');
  }
})();