derma/src/scrape.js

const { pathOr } = require('ramda');

const {
  create_images_directory,
  get_upcoming_events,
  open_browser,
  parse_args,
  read_previous_events,
} = require('./logic');

const { page_ids, output, events: event_file } = parse_args(
  process.argv.slice(2),
);

(async () => {
  create_images_directory('./img');

  const previous_events = await read_previous_events(event_file);

  const browser = await open_browser();

  let events = [];

  for (let page_id of page_ids) {
    const new_events = get_upcoming_events(browser, page_id);

    events = merge_edges(new_events, events);
    events = events.filter(
      ({ event_id }) =>
        previous_events.find(
          (previous_event) => event_id === previous_event.event_id,
        ) === undefined,
    );

    events = await Promise.all(
      events.map(async (event) => {
        const event_page = await browser.newPage();
        const event_data = await load_event(event_page, event.event_id);
        return {
          ...event_data,
          ...event,
        };
      }),
    );
  }

  events = await Promise.all(
    events.map(async (event) => {
      const images = await save_images(event);
      delete event.image;
      return {
        images,
        ...event,
      };
    }),
  );

  if (output === null) {
    const all_events = [...events, ...previous_events]
      .map((event) => {
        const start = pathOr(null, ['date', 'start'], event);
        if (start !== null) {
          try {
            event.date.start = new Date(start);
          } catch (e) {
            console.error(e);
            return event;
          }
        }
        return event;
      })
      .sort((a, b) => {
        const b_date = b.date.start;
        const a_date = a.date.start;
        if (a_date > b_date) {
          return 1;
        }
        if (a_date == b_date) {
          return 0;
        }
        return -1;
      });

    console.log(JSON.stringify(all_events));
  }

  process.exit();
})();