Browse Source

simplify

fix-broken-scrape
Jørgen Sverre Lien Sellæg 4 years ago
parent
commit
fcf6ee1884
  1. 148
      src/darma.mjs

148
src/darma.mjs

@ -1,94 +1,40 @@
import { hasPath, pathOr, props, last } from 'ramda';
import { do_request } from './facebook-request.mjs'; import { do_request } from './facebook-request.mjs';
const sleep = (s) => new Promise((res) => setTimeout(res, s * 1000)); const sleep = (s) => new Promise((res) => setTimeout(res, s * 1000));
const doPageEventsTabPastEventsCardRendererQuery = async ({ pageID }) => { /// PageEventsTabPastEventsCardRendererQuery
export const past_render_query = async ({ pageID }) => {
const doc_id = '4421910857857782'; const doc_id = '4421910857857782';
const renderer_query_result = await do_request(doc_id, { pageID }); const resp = await do_request(doc_id, { pageID });
const page = pathOr( const page = resp?.data?.page?.past_events ?? null;
null,
['data', 'page', 'past_events'],
renderer_query_result
);
if (page === null) {
console.error('doPageEventsTabPastEventsCardRendererQuery returned null.');
}
return page; return page;
}; };
const doPageEventsTabUpcomingEventsCardRendererQuery = async ({ pageID }) => { /// PageEventsTabPastEventsCardPaginationQuery
const doc_id = '5182274978466320'; export const past_pagination_query = async ({ pageID, cursor }) => {
const renderer_query_result = await do_request(doc_id, { pageID }); const doc_id = '6953034388071359';
const page = pathOr( const count = 9;
null, const resp = await do_request(doc_id, { pageID, cursor, count });
['data', 'page', 'upcoming_events'], const page = resp?.data?.page?.past_events ?? null;
renderer_query_result
);
if (page === null) {
console.error(
'doPageEventsTabUpcomingEventsCardRendererQuery returned null.'
);
}
return page; return page;
}; };
const doPageEventsTabPastEventsCardPaginationQuery = async ({ /// PageEventsTabUpcomingEventsCardRendererQuery
pageID, export const upcoming_render_query = async ({ pageID }) => {
cursor const doc_id = '5182274978466320';
}) => { const resp = await do_request(doc_id, { pageID });
const doc_id = '6953034388071359'; const page = resp?.data?.page?.upcoming_events ?? null;
let count = 9;
const renderer_query_result = await do_request(doc_id, {
pageID,
cursor,
count
});
const page = pathOr(
null,
['data', 'page', 'past_events'],
renderer_query_result
);
if (page === null) {
console.error(
'doPageEventsTabPastEventsCardPaginationQuery returned null.'
);
}
return page; return page;
}; };
const doPageEventsTabUpcomingEventsCardPaginationQuery = async ({ // PageEventsTabUpcomingEventsCardPaginationQuery
pageID, export const upcoming_pagination_query = async ({ pageID, cursor }) => {
cursor
}) => {
const doc_id = '6985622308176123'; const doc_id = '6985622308176123';
let count = 9; let count = 9;
const renderer_query_result = await do_request(doc_id, { const resp = await do_request(doc_id, { pageID, cursor, count });
pageID, const page = resp?.data?.page?.upcoming_events ?? null;
cursor,
count
});
const page = pathOr(
null,
['data', 'page', 'upcoming_events'],
renderer_query_result
);
if (page === null) {
console.error(
'doPageEventsTabUpcomingEventsCardPaginationQuery returned null.'
);
}
return page; return page;
}; };
// doPageEventsTabPastEventsCardPaginationQuery
const get_page_events = async ({ const get_page_events = async ({
pageID, pageID,
get_past_events, get_past_events,
@ -97,7 +43,7 @@ const get_page_events = async ({
let past_events = []; let past_events = [];
if (get_past_events) { if (get_past_events) {
const result = await doPageEventsTabPastEventsCardRendererQuery({ pageID }); const result = await past_render_query({ pageID });
if (result !== null) { if (result !== null) {
let { has_next_page, end_cursor: cursor } = result.page_info; let { has_next_page, end_cursor: cursor } = result.page_info;
@ -105,16 +51,15 @@ const get_page_events = async ({
let { edges } = result; let { edges } = result;
while (has_next_page) { while (has_next_page) {
sleep(2); sleep(2);
const paginationResult = const paginationResult = await past_pagination_query({
await doPageEventsTabPastEventsCardPaginationQuery({ cursor,
cursor, pageID
pageID });
});
if (paginationResult === null) { if (paginationResult === null) {
break; break;
} }
edges = [...edges, ...paginationResult.edges]; edges = [...edges, ...paginationResult.edges];
has_next_page = paginationResult.page_info.has_next_page; has_next_page = paginationResult?.page_info?.has_next_page ?? false;
cursor = paginationResult.page_info.end_cursor; cursor = paginationResult.page_info.end_cursor;
} }
past_events = [...edges]; past_events = [...edges];
@ -124,27 +69,23 @@ const get_page_events = async ({
let upcoming_events = []; let upcoming_events = [];
if (get_upcoming_events) { if (get_upcoming_events) {
const result = await doPageEventsTabUpcomingEventsCardRendererQuery({ const result = await upcoming_render_query({ pageID });
pageID
});
if (result !== null) { if (result !== null) {
upcoming_events = []; upcoming_events = [];
let { has_next_page, end_cursor: cursor } = result.page_info; let { has_next_page, end_cursor: cursor } = result.page_info;
let { edges } = result; let { edges } = result;
while (has_next_page) { while (has_next_page) {
sleep(2); sleep(2);
const paginationResult = const paginationResult = await upcoming_pagination_query({
await doPageEventsTabUpcomingEventsCardPaginationQuery({ cursor,
cursor, pageID
pageID });
});
if (paginationResult === null) { if (paginationResult === null) {
break; break;
} }
edges = [...edges, ...paginationResult.edges]; edges = [...edges, ...paginationResult.edges];
has_next_page = paginationResult?.page_info?.has_next_page ?? false;
has_next_page = paginationResult.page_info.has_next_page; cursor = paginationResult?.page_info?.end_cursor;
cursor = paginationResult.page_info.end_cursor;
} }
upcoming_events = [...edges]; upcoming_events = [...edges];
} }
@ -153,27 +94,4 @@ const get_page_events = async ({
return [...upcoming_events, ...past_events].map(({ node }) => node); return [...upcoming_events, ...past_events].map(({ node }) => node);
}; };
/// const events = { export default get_page_events;
/// edges: [
/// {
/// node: {
/// __typename: "Event",
/// },
/// cursor:
/// "AQHRC7ZNKEqDS75jWJfLUWromnLVgAOGzVAZE7c7CcKfoEaLCcXFSvhMvoxN8yk_Yq6fFMTWjuHjitD5sE1IzW68sw",
/// },
/// ],
/// page_info: {
/// has_next_page: true,
/// end_cursor:
/// "AQHRAh7tKZowf3mdyxtYISP1LNVo45rFI8HJ4nT5SuVgl0NBUfZFslx5qy1eba3YXhdjJ-S2vfojcTGF4ygnt_DQiQ",
/// },
/// };
(async () => {
const res = await get_page_events({
pageID: '149127815110411',
get_upcoming_events: true
});
console.log(res);
})();

Loading…
Cancel
Save