Browse Source

add output and input

fix-broken-scrape
Jørgen Sverre Lien Sellæg 5 years ago
parent
commit
7e410ce6d9
  1. 24
      src/scrape-event.js

24
src/scrape-event.js

@ -1,4 +1,9 @@
const { parse_args, sleep } = require('./logic'); const {
parse_args,
sleep,
read_previous_events,
write_events,
} = require('./logic');
const { get_event_details } = require('./facebook-event.js'); const { get_event_details } = require('./facebook-event.js');
const { get_page_events } = require('./facebook-page-events.js'); const { get_page_events } = require('./facebook-page-events.js');
@ -8,14 +13,20 @@ const { omit } = require('ramda');
const options = parse_args(process.argv.slice(2)); const options = parse_args(process.argv.slice(2));
(async () => { (async () => {
const previous_events = await read_previous_events(options.events);
const page_events = await get_page_events(options); const page_events = await get_page_events(options);
let events = []; let events = [];
for (const event of page_events) { for (const event of page_events) {
const index = previous_events.findIndex(({ id }) => event.id === id);
if (index === -1) {
sleep(2); sleep(2);
const event_details = await get_event_details(event.id); const event_details = await get_event_details(event.id);
events.push({ ...event, ...event_details }); events.push({ ...event, ...event_details });
} else {
events.push(previous_events[index]);
}
} }
const filterAwayFields = omit([ const filterAwayFields = omit([
@ -47,5 +58,16 @@ const options = parse_args(process.argv.slice(2));
events = events.map(filterAwayFields); events = events.map(filterAwayFields);
if (options.output === null) {
console.log(JSON.stringify(events)); console.log(JSON.stringify(events));
process.exit();
}
try {
await write_events(options.output, events);
process.exit();
} catch (e) {
console.error(e);
process.exit(1);
}
})(); })();

Loading…
Cancel
Save