derma/src/scrape.js

const { pathOr, uniqBy, eqBy, prop, union } = require('ramda');

const {
  create_images_directory,
  get_page_events,
  open_browser,
  parse_args,
  read_previous_events,
  merge_edges,
  load_event,
  save_images,
} = require('./logic');

const {
  events: event_file,
  get_past_events,
  get_upcoming_events,
  image_directory,
  images,
  output,
  page_ids,
  headless,
} = parse_args(process.argv.slice(2));

(async () => {
  if (images) {
    create_images_directory(image_directory);
  }

  let events = [];

  const browser = await open_browser({ headless });

  for (let page_id of page_ids) {
    let page_events = [];
    try {
      page_events = await get_page_events(
        browser,
        page_id,
        get_upcoming_events,
        get_past_events,
      );
    } catch (e) {
      console.error(e);
    }
    if (images) {
      page_events = await Promise.all(
        page_events.map(async (event) => {
          const event_page = await browser.newPage();
          const { image } = await load_event(event_page, event.event_id);
          await event_page.close();
          const images = await save_images(
            image,
            event.event_id,
            image_directory,
          );
          return { images, ...event };
        }),
      );
    }

    events = uniqBy(eqBy(prop('event_id')))(union(events, page_events));
  }

  const previous_events = await read_previous_events(event_file);

  let all_events = events
    .reduce(merge_edges, previous_events)
    .map((event) => {
      const start = pathOr(null, ['date', 'start'], event);
      if (start !== null) {
        try {
          event.date.start = new Date(start);
        } catch (e) {
          console.error(e);
          return event;
        }
      }
      return event;
    })
    .sort((a, b) => {
      const b_date = b.date.start;
      const a_date = a.date.start;
      if (a_date > b_date) {
        return 1;
      }
      if (a_date == b_date) {
        return 0;
      }
      return -1;
    });

  if (output === null) {
    console.log(JSON.stringify(all_events));
    process.exit();
  }
})();