derma/src/scrape.js

const { pathOr } = require('ramda');

const {
  create_images_directory,
  get_page_events,
  open_browser,
  parse_args,
  read_previous_events,
  merge_edges,
  load_event,
  save_images,
} = require('./logic');

const {
  events: event_file,
  image_directory,
  images,
  output,
  page_ids,
} = parse_args(process.argv.slice(2));

(async () => {
  if (images) {
    create_images_directory(image_directory);
  }

  const previous_events = await read_previous_events(event_file);
  const browser = await open_browser();

  let events = [];

  for (let page_id of page_ids) {
    const past_events = false;
    const new_events = await get_page_events(
      browser,
      page_id,
      true,
      past_events,
    );

    events = merge_edges(new_events, events);
    events = events.filter(
      ({ event_id }) =>
        previous_events.find(
          (previous_event) => event_id === previous_event.event_id,
        ) === undefined,
    );
    if (images) {
      events = await Promise.all(
        events.map(async (event) => {
          const event_page = await browser.newPage();
          const { image } = await load_event(event_page, event.event_id);
          event_page.close();
          const images = await save_images(image, event.event_id);
          return {
            images,
            ...event,
          };
        }),
      );
    }
  }

  let all_events = merge_edges(events, previous_events)
    .map((event) => {
      const start = pathOr(null, ['date', 'start'], event);
      if (start !== null) {
        try {
          event.date.start = new Date(start);
        } catch (e) {
          console.error(e);
          return event;
        }
      }
      return event;
    })
    .sort((a, b) => {
      const b_date = b.date.start;
      const a_date = a.date.start;
      if (a_date > b_date) {
        return 1;
      }
      if (a_date == b_date) {
        return 0;
      }
      return -1;
    });

  if (output === null) {
    console.log(JSON.stringify(all_events));
    process.exit();
  }
})();