Browse Source

refactor fetching of events to make use of the initial request

fix-broken-scrape
Jørgen Sverre Lien Sellæg 5 years ago
parent
commit
987f69f3f7
  1. 108
      src/facebook-api.js

108
src/facebook-api.js

@ -11,21 +11,6 @@ const fetch = require('node-fetch');
const last = require('ramda/src/last');
const fetch_past_events = async (page_id, cursor = null) => {
let doc_id;
if (cursor === null) {
doc_id = '3636086023161977';
} else {
doc_id = '3972530646159673';
}
const res = await do_events_request(page_id, doc_id, cursor);
return get_past_events_from_page(res);
};
const get_past_events = async () => [];
const do_events_request = async (page_id, doc_id, cursor = null) => {
const params = new URLSearchParams();
@ -73,50 +58,91 @@ const do_events_request = async (page_id, doc_id, cursor = null) => {
return res;
};
const fetch_upcoming_events = async (page_id, cursor = null) => {
let doc_id;
if (cursor === null) {
doc_id = '3636086023161977';
} else {
doc_id = '3911675102281316';
}
const res = await do_events_request(page_id, doc_id, cursor);
return get_upcoming_events_from_page(res);
};
const get_upcoming_events = async (page_id) => {
let next = true;
let cursor;
let edges = [];
const get_events = async (page_id, doc_id, get_events_from_page, events) => {
let { edges, cursor } = events;
let next = cursor !== null;
while (next) {
const res = await fetch_upcoming_events(page_id, cursor);
edges = [...edges, ...get_edges(res)];
const { has_next_page } = get_page_info(res);
let page;
page = await do_events_request(page_id, doc_id, cursor);
const { has_next_page } = get_page_info(page);
page = get_events_from_page(page);
edges = [...edges, ...get_edges(page)];
next = has_next_page;
cursor = last(edges).cursor;
if (has_next_page === true) {
if (next) {
await sleep(2);
}
}
return edges;
};
const get_upcoming_events = async (page_id, events) => {
const doc_id = '3911675102281316';
const upcoming_events = await get_events(
page_id,
doc_id,
get_upcoming_events_from_page,
events,
);
return upcoming_events;
};
const get_past_events = async (page_id, events) => {
const doc_id = '3972530646159673';
const past_events = await get_events(
page_id,
doc_id,
get_past_events_from_page,
events,
);
return past_events;
};
const get_reoccuring_events = () => {};
const init_scrape = async (page_id) => {
const doc_id = '3636086023161977';
const res = await do_events_request(page_id, doc_id);
const upcoming_events = get_upcoming_events_from_page(res);
const past_events = get_past_events_from_page(res);
const upcoming_edges = get_edges(upcoming_events);
const past_edges = get_edges(past_events);
const upcoming_has_next_page = get_page_info(upcoming_events).has_next_page;
const past_has_next_page = get_page_info(past_events).has_next_page;
return {
upcoming_events: {
edges: upcoming_edges,
cursor: upcoming_has_next_page ? last(upcoming_edges).cursor : null,
},
past_events: {
cursor: past_has_next_page ? last(past_edges).cursor : null,
edges: past_edges,
},
};
};
const get_page_events = async (opt) => {
let past_events = [];
let upcoming_events = [];
const { page_id } = opt;
let { upcoming_events, past_events } = await init_scrape(page_id);
if (opt.get_past_events) {
past_events = await get_past_events(page_id);
await sleep(2);
past_events = await get_past_events(page_id, past_events);
} else {
past_events = [];
}
if (opt.get_upcoming_events) {
upcoming_events = await get_upcoming_events(page_id);
await sleep(2);
upcoming_events = await get_upcoming_events(page_id, upcoming_events);
} else {
upcoming_events = [];
}
const nodes = [...upcoming_events, ...past_events];

Loading…
Cancel
Save