scrape reocurring events

4 years ago · ca5964a558
4 changed files with 136 additions and 90 deletions
--- a/bin/adapter.mjs
+++ b/bin/adapter.mjs
@ -42,7 +42,7 @@ export const unix = (a) => parseInt(new Date(a).valueOf() / 1000, 10);
  for (let i = 0; i < requests.length; i++) {
    let event = requests[i];
    const res = await fetch(
-      'http://localhost:3333/events/?token=831411806230c7e950c4eeb226499ef92bb6bdc4157797929a0e16d133dc13a8',
+      'https://dansbart.no/events/?token=831411806230c7e950c4eeb226499ef92bb6bdc4157797929a0e16d133dc13a8',
      {
        method: 'POST',
        body: JSON.stringify(event),
--- a/bin/run.sh
+++ b/bin/run.sh
@ -1,82 +1,97 @@
 #!/usr/bin/env bash
+readonly SCRIPT_HOME=$(dirname `readlink -f $0`)
+export NODE_EXTRA_CA_CERTS="$SCRIPT_HOME/../share/ca.crt"
+
+
+function scrape_page {
+  local page_id;
+  local storage_path;
+  page_id="$1"
+  storage_path="$2"
+
+  echo "Scraping $storage_path";
+  bin/scrape $page_id > $storage_path;
+}
+
 mkdir -p events
-# bin/scrape 145869695444569 >> events/AftermathMusicNorway-145869695444569.json
-# bin/scrape 774453762576345 >> events/BarnefestivalenJubaJuba-774453762576345.json
-bin/scrape 750972958283466 >> events/Boxerbar-750972958283466.old.json
-bin/scrape 153799614679194 >> events/Britannia-153799614679194.Hotel.old.json
-bin/scrape 380071843764 >> events/ByscenenTrondheim-380071843764.old.json
-bin/scrape 220501644709429 >> events/CinemateketTrondheim-220501644709429.old.json
-bin/scrape 225737590803212 >> events/Diskoteketbar-225737590803212.old.json
-bin/scrape 325557631163470 >> events/Dora3Trondheim-325557631163470.old.json
-bin/scrape 784700614931033 >> events/Goodomensbar-784700614931033.old.json
-bin/scrape 1673051982745157 >> events/HjortenScene-1673051982745157.old.json
-bin/scrape 11541693212 >> events/ISFiTNorway-11541693212.old.json
-bin/scrape 1419662314995709 >> events/KafeSkuret-1419662314995709.old.json
-bin/scrape 1092418890774152 >> events/Kafelarssen-1092418890774152.old.json
-bin/scrape 154134355099614 >> events/KonsertGalleriet-154134355099614.old.json
-bin/scrape 2198343290262409.trondheim >> events/Lager11-2198343290262409.trondheim.old.json
-bin/scrape 287716286232 >> events/Pstereo-287716286232.old.json
-bin/scrape 111985600537696 >> events/Sommer-i-Borggården-111985600537696-111985600537696.old.json
-bin/scrape 151868258198492 >> events/Sukkerhuset-151868258198492.old.json
-bin/scrape 274218068628 >> events/TRONDHEIMISAK-274218068628.old.json
-bin/scrape 705753499633189 >> events/TrondheimCamping-705753499633189.old.json
-bin/scrape 214229302015789 >> events/TrondheimJazzforum-214229302015789.old.json
-bin/scrape 311676952866578 >> events/TrondheimLive-311676952866578.old.json
-bin/scrape 114647705270950 >> events/TrondheimMetalFest-114647705270950.old.json
-bin/scrape 1866856830009864 >> events/TrondheimRocks-1866856830009864.old.json
-bin/scrape 175439175828277 >> events/TrondheimSpektrum-175439175828277.old.json
-bin/scrape 108747454194270 >> events/TrykkerietScene-108747454194270.old.json
-bin/scrape 626731467352429 >> events/WorkWorkTrd-626731467352429.old.json
-bin/scrape 300444313497473 >> events/almasrestaurant-300444313497473.old.json
-bin/scrape 106346922773023 >> events/antikvarene-106346922773023.old.json
-bin/scrape 150118755488 >> events/artscenetrondheim-150118755488.old.json
-bin/scrape 158501780863403 >> events/barcircus-158501780863403.old.json
-bin/scrape 160806787399376 >> events/barmoskus-160806787399376.old.json
-bin/scrape 1712859978926496 >> events/brorbaras-1712859978926496.old.json
-bin/scrape 191072824261732 >> events/cafenim-191072824261732.old.json
-bin/scrape 129738423764486 >> events/cafenimuser-129738423764486.old.json
-bin/scrape 118990964789505 >> events/clarioncollectiongrandolav-118990964789505.old.json
-bin/scrape 148274751888665 >> events/clariontrondheim-148274751888665.old.json
-bin/scrape 378038755927834 >> events/comforthpark-378038755927834.old.json
-bin/scrape 271908336649895 >> events/crispingloverrecordshop-271908336649895.old.json
-bin/scrape 860188704010725 >> events/feminalen-860188704010725.old.json
-bin/scrape 654277341296114 >> events/fru-654277341296114.lundgreen.old.json
-bin/scrape 955982084466587 >> events/habitattrondheim-955982084466587.old.json
-bin/scrape 143110949094001 >> events/ilabrainnstasjon-143110949094001.old.json
-bin/scrape 105916402763134 >> events/jazzfestno-105916402763134.old.json
-bin/scrape 80792973478 >> events/kamfest-80792973478.old.json
-bin/scrape 137220573544106 >> events/komikerliv-137220573544106.old.json
-bin/scrape 1402925659778914 >> events/konsertkollektiv-1402925659778914.old.json
-bin/scrape 642243462457769 >> events/kulturnattrondheim-642243462457769.old.json
-bin/scrape 246603018828671 >> events/kunsthalltrondheim-246603018828671.old.json
-bin/scrape 180524565650597 >> events/ladekaia-180524565650597.old.json
-bin/scrape 1552366408164455.scenen >> events/lamon-1552366408164455.scenen.old.json
-bin/scrape 101846779891549 >> events/lobbyen-101846779891549.old.json
-bin/scrape 191245541651071.trhm >> events/lokal-191245541651071.trhm.old.json
-bin/scrape 346065342451673 >> events/midtbyenrockeklubb-346065342451673.old.json
-bin/scrape 1786002618299415 >> events/nidelvenbarogscene-1786002618299415.old.json
-bin/scrape 90016673206 >> events/olavsfest-90016673206.old.json
-bin/scrape 149127815110411 >> events/olavshallenas-149127815110411.old.json
-bin/scrape 130558850296573 >> events/qualityhotelpanorama-130558850296573.old.json
-bin/scrape 286956505871 >> events/ringvemusikkmuseum-286956505871.old.json
-bin/scrape 68623573578 >> events/rockheim-68623573578.old.json
-bin/scrape 11782406887 >> events/samfundet-11782406887.old.json
-bin/scrape 375949715790210 >> events/scandicbakklandet-375949715790210.old.json
-bin/scrape 260116090806604 >> events/scandiclerkendal-260116090806604.old.json
-bin/scrape 127377617334855 >> events/scandicnidelven-127377617334855.old.json
-bin/scrape 799107733481572 >> events/scandicsolsiden-799107733481572.old.json
-bin/scrape 169539553659073.trondheim >> events/scenekanten-169539553659073.trondheim.old.json
-bin/scrape 110148939061528 >> events/solsidentrondheim-110148939061528.old.json
-bin/scrape 109571644107406 >> events/stormfestivalen-109571644107406.old.json
-bin/scrape 484033441709379 >> events/stundomrecords-484033441709379.old.json
-bin/scrape 208801362485221 >> events/thonhotelprinsen-208801362485221.old.json
-bin/scrape 1649281185339329 >> events/trollrestaurant-1649281185339329.old.json
-bin/scrape 33687748452 >> events/trondercore-33687748452.old.json
-bin/scrape 112475092130212.kunstmuseum >> events/trondheim-112475092130212.kunstmuseum.old.json
-bin/scrape 192236567852623 >> events/trondheimbluesklubb-192236567852623.old.json
-bin/scrape 125977737456625 >> events/trondheimcalling-125977737456625.old.json
-bin/scrape 16582146804 >> events/trondheimfolkebibliotek-16582146804.old.json
-bin/scrape 1858914204185866 >> events/trondheimstage-1858914204185866.old.json
-bin/scrape 237410323543222 >> events/tyventrondheim-237410323543222.old.json
-bin/scrape 697776233588499 >> events/uffabookingcrew-697776233588499.old.json
-bin/scrape 965958530110723 >> events/vaarfruekirke-965958530110723.old.json
+# scrape_page 145869695444569 events/AftermathMusicNorway-145869695444569.json
+# scrape_page 774453762576345 events/BarnefestivalenJubaJuba-774453762576345.json
+# scrape_page 1633600613619431 events/Hammerhead-1633600613619431.json
+# scrape_page 750972958283466 events/Boxerbar-750972958283466.json
+# scrape_page 153799614679194 events/Britannia-153799614679194.Hotel.json
+# scrape_page 380071843764 events/ByscenenTrondheim-380071843764.json
+# scrape_page 220501644709429 events/CinemateketTrondheim-220501644709429.json
+# scrape_page 225737590803212 events/Diskoteketbar-225737590803212.json
+# scrape_page 325557631163470 events/Dora3Trondheim-325557631163470.json
+# scrape_page 784700614931033 events/Goodomensbar-784700614931033.json
+# scrape_page 1673051982745157 events/HjortenScene-1673051982745157.json
+# scrape_page 11541693212 events/ISFiTNorway-11541693212.json
+# scrape_page 1419662314995709 events/KafeSkuret-1419662314995709.json
+# scrape_page 1092418890774152 events/Kafelarssen-1092418890774152.json
+# scrape_page 154134355099614 events/KonsertGalleriet-154134355099614.json
+# scrape_page 2198343290262409 events/Lager11-2198343290262409.trondheim.json
+# scrape_page 287716286232 events/Pstereo-287716286232.json
+# scrape_page 111985600537696 events/Sommer-i-Borggården-111985600537696-111985600537696.json
+# scrape_page 151868258198492 events/Sukkerhuset-151868258198492.json
+# scrape_page 274218068628 events/TRONDHEIMISAK-274218068628.json
+# scrape_page 705753499633189 events/TrondheimCamping-705753499633189.json
+# scrape_page 214229302015789 events/TrondheimJazzforum-214229302015789.json
+# scrape_page 311676952866578 events/TrondheimLive-311676952866578.json
+# scrape_page 114647705270950 events/TrondheimMetalFest-114647705270950.json
+# scrape_page 1866856830009864 events/TrondheimRocks-1866856830009864.json
+# scrape_page 175439175828277 events/TrondheimSpektrum-175439175828277.json
+# scrape_page 108747454194270 events/TrykkerietScene-108747454194270.json
+# scrape_page 626731467352429 events/WorkWorkTrd-626731467352429.json
+# scrape_page 300444313497473 events/almasrestaurant-300444313497473.json
+# scrape_page 106346922773023 events/antikvarene-106346922773023.json
+# scrape_page 150118755488 events/artscenetrondheim-150118755488.json
+# scrape_page 158501780863403 events/barcircus-158501780863403.json
+# scrape_page 160806787399376 events/barmoskus-160806787399376.json
+# scrape_page 1712859978926496 events/brorbaras-1712859978926496.json
+# scrape_page 191072824261732 events/cafenim-191072824261732.json
+# scrape_page 129738423764486 events/cafenimuser-129738423764486.json
+# scrape_page 118990964789505 events/clarioncollectiongrandolav-118990964789505.json
+# scrape_page 148274751888665 events/clariontrondheim-148274751888665.json
+# scrape_page 378038755927834 events/comforthpark-378038755927834.json
+# scrape_page 271908336649895 events/crispingloverrecordshop-271908336649895.json
+# scrape_page 860188704010725 events/feminalen-860188704010725.json
+# scrape_page 654277341296114 events/fru-654277341296114.lundgreen.json
+# scrape_page 955982084466587 events/habitattrondheim-955982084466587.json
+# scrape_page 143110949094001 events/ilabrainnstasjon-143110949094001.json
+# scrape_page 105916402763134 events/jazzfestno-105916402763134.json
+# scrape_page 80792973478 events/kamfest-80792973478.json
+# scrape_page 137220573544106 events/komikerliv-137220573544106.json
+# scrape_page 1402925659778914 events/konsertkollektiv-1402925659778914.json
+# scrape_page 642243462457769 events/kulturnattrondheim-642243462457769.json
+# scrape_page 246603018828671 events/kunsthalltrondheim-246603018828671.json
+# scrape_page 180524565650597 events/ladekaia-180524565650597.json
+# scrape_page 1552366408164455 events/lamon-1552366408164455.scenen.json
+# scrape_page 101846779891549 events/lobbyen-101846779891549.json
+# scrape_page 191245541651071 events/lokal-191245541651071.trhm.json
+# scrape_page 346065342451673 events/midtbyenrockeklubb-346065342451673.json
+# scrape_page 1786002618299415 events/nidelvenbarogscene-1786002618299415.json
+# scrape_page 90016673206 events/olavsfest-90016673206.json
+# scrape_page 149127815110411 events/olavshallenas-149127815110411.json
+# scrape_page 130558850296573 events/qualityhotelpanorama-130558850296573.json
+# scrape_page 286956505871 events/ringvemusikkmuseum-286956505871.json
+# scrape_page 68623573578 events/rockheim-68623573578.json
+# scrape_page 11782406887 events/samfundet-11782406887.json
+# scrape_page 375949715790210 events/scandicbakklandet-375949715790210.json
+# scrape_page 260116090806604 events/scandiclerkendal-260116090806604.json
+# scrape_page 127377617334855 events/scandicnidelven-127377617334855.json
+# scrape_page 799107733481572 events/scandicsolsiden-799107733481572.json
+# scrape_page 169539553659073 events/scenekanten-169539553659073.trondheim.json
+# scrape_page 110148939061528 events/solsidentrondheim-110148939061528.json
+# scrape_page 109571644107406 events/stormfestivalen-109571644107406.json
+# scrape_page 484033441709379 events/stundomrecords-484033441709379.json
+# scrape_page 208801362485221 events/thonhotelprinsen-208801362485221.json
+# scrape_page 1649281185339329 events/trollrestaurant-1649281185339329.json
+# scrape_page 33687748452 events/trondercore-33687748452.json
+# scrape_page 112475092130212 events/trondheim-112475092130212.kunstmuseum.json
+# scrape_page 192236567852623 events/trondheimbluesklubb-192236567852623.json
+# scrape_page 125977737456625 events/trondheimcalling-125977737456625.json
+# scrape_page 16582146804 events/trondheimfolkebibliotek-16582146804.json
+# scrape_page 1858914204185866 events/trondheimstage-1858914204185866.json
+# scrape_page 237410323543222 events/tyventrondheim-237410323543222.json
+# scrape_page 697776233588499 events/uffabookingcrew-697776233588499.json
+scrape_page 965958530110723 events/vaarfruekirke-965958530110723.json
--- a/bin/scrape
+++ b/bin/scrape
@ -14,8 +14,8 @@ if (number == null) {
  try {
    const res = await get_page_events({
      pageID: number,
-      get_upcoming_events: false,
-      get_past_events: true
+      get_upcoming_events: true,
+      get_past_events: false
    });
    console.log(JSON.stringify(res));
  } catch (e) {
--- a/src/facebook/get-page-events.mjs
+++ b/src/facebook/get-page-events.mjs
@ -35,12 +35,41 @@ export const upcoming_pagination_query = async ({ pageID, cursor }) => {
  return page;
 };

+// PageEventsTabRecurringEventsCardRendererQuery
+export const upcoming_reoccuring_render_query = async ({ pageID }) => {
+  const doc_id = '4458618790924188';
+  const resp = await do_request(doc_id, { pageID });
+  const page = resp?.data?.page?.upcomingRecurringEvents?.edges ?? null;
+
+  if (page === null || !Array.isArray(page)) {
+    return null;
+  }
+
+  let res = [];
+
+  page.forEach(({ node }) => {
+    let { childEvents, ...parent } = node;
+    childEvents.edges.forEach(({ node }) => {
+      let event = {
+        ...parent,
+        ...{
+          id: node.id,
+          time_range: { start: new Date(node.currentStartTimestamp * 1000) }
+        }
+      };
+      res.push(event);
+    });
+  });
+
+  return res;
+};
+
 export const get_page_events = async ({
  pageID,
  get_past_events,
  get_upcoming_events
 }) => {
-  let past_events = [];
+  let res = [];

  if (get_past_events) {
    const result = await past_render_query({ pageID });
@ -68,16 +97,13 @@ export const get_page_events = async ({
        has_next_page = paginationResult?.page_info?.has_next_page ?? false;
        cursor = paginationResult.page_info.end_cursor;
      }
-      past_events = [...edges];
+      res = [...edges.map(({ node }) => node)];
    }
  }

-  let upcoming_events = [];
-
  if (get_upcoming_events) {
    const result = await upcoming_render_query({ pageID });
    if (result !== null) {
-      upcoming_events = [];
      let { has_next_page, end_cursor: cursor } = result.page_info;
      let { edges } = result;
      let retries = 0;
@ -98,11 +124,16 @@ export const get_page_events = async ({
        has_next_page = paginationResult?.page_info?.has_next_page ?? false;
        cursor = paginationResult?.page_info?.end_cursor;
      }
-      upcoming_events = [...edges];
+
+      res = [...edges.map(({ node }) => node)];
+    }
+    const reoccuring = await upcoming_reoccuring_render_query({ pageID });
+    if (reoccuring !== null) {
+      res = [...res, ...reoccuring];
    }
  }

-  return [...upcoming_events, ...past_events].map(({ node }) => node);
+  return res;
 };

 export default get_page_events;