derma/src/scrape.js

const { pathOr, uniqBy, eqBy, prop, union } = require('ramda');
const { merge_edges, parse_args, read_previous_events } = require('./logic');
const { open_browser, get_page_events } = require('./browser');

const {
  events: event_file,
  get_past_events,
  get_upcoming_events,
  output,
  page_ids,
  headless,
} = parse_args(process.argv.slice(2));

(async () => {
  let events = [];
  try {
    const browser = await open_browser({ headless });
    for (let page_id of page_ids) {
      let page_events = [];
      page_events = await get_page_events(
        browser,
        page_id,
        get_upcoming_events,
        get_past_events,
      );
      events = uniqBy(eqBy(prop('event_id')))(union(events, page_events));
    }
  } catch (e) {
    console.error(e);
  }

  const previous_events = await read_previous_events(event_file);

  let all_events = events
    .reduce(merge_edges, previous_events)
    .map((event) => {
      const start = pathOr(null, ['date', 'start'], event);
      if (start !== null) {
        try {
          event.date.start = new Date(start);
        } catch (e) {
          console.error(e);
          return event;
        }
      }
      return event;
    })
    .sort((a, b) => {
      const b_date = b.date.start;
      const a_date = a.date.start;
      if (a_date > b_date) {
        return 1;
      }
      if (a_date == b_date) {
        return 0;
      }
      return -1;
    });

  if (output === null) {
    console.log(JSON.stringify(all_events));
    process.exit();
  }
})();