diff --git a/bin/adapter.mjs b/bin/adapter.mjs index e7d98a0..d23541a 100644 --- a/bin/adapter.mjs +++ b/bin/adapter.mjs @@ -42,7 +42,7 @@ export const unix = (a) => parseInt(new Date(a).valueOf() / 1000, 10); for (let i = 0; i < requests.length; i++) { let event = requests[i]; const res = await fetch( - 'http://localhost:3333/events/?token=831411806230c7e950c4eeb226499ef92bb6bdc4157797929a0e16d133dc13a8', + 'https://dansbart.no/events/?token=831411806230c7e950c4eeb226499ef92bb6bdc4157797929a0e16d133dc13a8', { method: 'POST', body: JSON.stringify(event), diff --git a/bin/run.sh b/bin/run.sh index 298d9e0..3c17991 100755 --- a/bin/run.sh +++ b/bin/run.sh @@ -1,82 +1,97 @@ #!/usr/bin/env bash +readonly SCRIPT_HOME=$(dirname `readlink -f $0`) +export NODE_EXTRA_CA_CERTS="$SCRIPT_HOME/../share/ca.crt" + + +function scrape_page { + local page_id; + local storage_path; + page_id="$1" + storage_path="$2" + + echo "Scraping $storage_path"; + bin/scrape $page_id > $storage_path; +} + mkdir -p events -# bin/scrape 145869695444569 >> events/AftermathMusicNorway-145869695444569.json -# bin/scrape 774453762576345 >> events/BarnefestivalenJubaJuba-774453762576345.json -bin/scrape 750972958283466 >> events/Boxerbar-750972958283466.old.json -bin/scrape 153799614679194 >> events/Britannia-153799614679194.Hotel.old.json -bin/scrape 380071843764 >> events/ByscenenTrondheim-380071843764.old.json -bin/scrape 220501644709429 >> events/CinemateketTrondheim-220501644709429.old.json -bin/scrape 225737590803212 >> events/Diskoteketbar-225737590803212.old.json -bin/scrape 325557631163470 >> events/Dora3Trondheim-325557631163470.old.json -bin/scrape 784700614931033 >> events/Goodomensbar-784700614931033.old.json -bin/scrape 1673051982745157 >> events/HjortenScene-1673051982745157.old.json -bin/scrape 11541693212 >> events/ISFiTNorway-11541693212.old.json -bin/scrape 1419662314995709 >> events/KafeSkuret-1419662314995709.old.json -bin/scrape 1092418890774152 >> events/Kafelarssen-1092418890774152.old.json -bin/scrape 154134355099614 >> events/KonsertGalleriet-154134355099614.old.json -bin/scrape 2198343290262409.trondheim >> events/Lager11-2198343290262409.trondheim.old.json -bin/scrape 287716286232 >> events/Pstereo-287716286232.old.json -bin/scrape 111985600537696 >> events/Sommer-i-Borggården-111985600537696-111985600537696.old.json -bin/scrape 151868258198492 >> events/Sukkerhuset-151868258198492.old.json -bin/scrape 274218068628 >> events/TRONDHEIMISAK-274218068628.old.json -bin/scrape 705753499633189 >> events/TrondheimCamping-705753499633189.old.json -bin/scrape 214229302015789 >> events/TrondheimJazzforum-214229302015789.old.json -bin/scrape 311676952866578 >> events/TrondheimLive-311676952866578.old.json -bin/scrape 114647705270950 >> events/TrondheimMetalFest-114647705270950.old.json -bin/scrape 1866856830009864 >> events/TrondheimRocks-1866856830009864.old.json -bin/scrape 175439175828277 >> events/TrondheimSpektrum-175439175828277.old.json -bin/scrape 108747454194270 >> events/TrykkerietScene-108747454194270.old.json -bin/scrape 626731467352429 >> events/WorkWorkTrd-626731467352429.old.json -bin/scrape 300444313497473 >> events/almasrestaurant-300444313497473.old.json -bin/scrape 106346922773023 >> events/antikvarene-106346922773023.old.json -bin/scrape 150118755488 >> events/artscenetrondheim-150118755488.old.json -bin/scrape 158501780863403 >> events/barcircus-158501780863403.old.json -bin/scrape 160806787399376 >> events/barmoskus-160806787399376.old.json -bin/scrape 1712859978926496 >> events/brorbaras-1712859978926496.old.json -bin/scrape 191072824261732 >> events/cafenim-191072824261732.old.json -bin/scrape 129738423764486 >> events/cafenimuser-129738423764486.old.json -bin/scrape 118990964789505 >> events/clarioncollectiongrandolav-118990964789505.old.json -bin/scrape 148274751888665 >> events/clariontrondheim-148274751888665.old.json -bin/scrape 378038755927834 >> events/comforthpark-378038755927834.old.json -bin/scrape 271908336649895 >> events/crispingloverrecordshop-271908336649895.old.json -bin/scrape 860188704010725 >> events/feminalen-860188704010725.old.json -bin/scrape 654277341296114 >> events/fru-654277341296114.lundgreen.old.json -bin/scrape 955982084466587 >> events/habitattrondheim-955982084466587.old.json -bin/scrape 143110949094001 >> events/ilabrainnstasjon-143110949094001.old.json -bin/scrape 105916402763134 >> events/jazzfestno-105916402763134.old.json -bin/scrape 80792973478 >> events/kamfest-80792973478.old.json -bin/scrape 137220573544106 >> events/komikerliv-137220573544106.old.json -bin/scrape 1402925659778914 >> events/konsertkollektiv-1402925659778914.old.json -bin/scrape 642243462457769 >> events/kulturnattrondheim-642243462457769.old.json -bin/scrape 246603018828671 >> events/kunsthalltrondheim-246603018828671.old.json -bin/scrape 180524565650597 >> events/ladekaia-180524565650597.old.json -bin/scrape 1552366408164455.scenen >> events/lamon-1552366408164455.scenen.old.json -bin/scrape 101846779891549 >> events/lobbyen-101846779891549.old.json -bin/scrape 191245541651071.trhm >> events/lokal-191245541651071.trhm.old.json -bin/scrape 346065342451673 >> events/midtbyenrockeklubb-346065342451673.old.json -bin/scrape 1786002618299415 >> events/nidelvenbarogscene-1786002618299415.old.json -bin/scrape 90016673206 >> events/olavsfest-90016673206.old.json -bin/scrape 149127815110411 >> events/olavshallenas-149127815110411.old.json -bin/scrape 130558850296573 >> events/qualityhotelpanorama-130558850296573.old.json -bin/scrape 286956505871 >> events/ringvemusikkmuseum-286956505871.old.json -bin/scrape 68623573578 >> events/rockheim-68623573578.old.json -bin/scrape 11782406887 >> events/samfundet-11782406887.old.json -bin/scrape 375949715790210 >> events/scandicbakklandet-375949715790210.old.json -bin/scrape 260116090806604 >> events/scandiclerkendal-260116090806604.old.json -bin/scrape 127377617334855 >> events/scandicnidelven-127377617334855.old.json -bin/scrape 799107733481572 >> events/scandicsolsiden-799107733481572.old.json -bin/scrape 169539553659073.trondheim >> events/scenekanten-169539553659073.trondheim.old.json -bin/scrape 110148939061528 >> events/solsidentrondheim-110148939061528.old.json -bin/scrape 109571644107406 >> events/stormfestivalen-109571644107406.old.json -bin/scrape 484033441709379 >> events/stundomrecords-484033441709379.old.json -bin/scrape 208801362485221 >> events/thonhotelprinsen-208801362485221.old.json -bin/scrape 1649281185339329 >> events/trollrestaurant-1649281185339329.old.json -bin/scrape 33687748452 >> events/trondercore-33687748452.old.json -bin/scrape 112475092130212.kunstmuseum >> events/trondheim-112475092130212.kunstmuseum.old.json -bin/scrape 192236567852623 >> events/trondheimbluesklubb-192236567852623.old.json -bin/scrape 125977737456625 >> events/trondheimcalling-125977737456625.old.json -bin/scrape 16582146804 >> events/trondheimfolkebibliotek-16582146804.old.json -bin/scrape 1858914204185866 >> events/trondheimstage-1858914204185866.old.json -bin/scrape 237410323543222 >> events/tyventrondheim-237410323543222.old.json -bin/scrape 697776233588499 >> events/uffabookingcrew-697776233588499.old.json -bin/scrape 965958530110723 >> events/vaarfruekirke-965958530110723.old.json +# scrape_page 145869695444569 events/AftermathMusicNorway-145869695444569.json +# scrape_page 774453762576345 events/BarnefestivalenJubaJuba-774453762576345.json +# scrape_page 1633600613619431 events/Hammerhead-1633600613619431.json +# scrape_page 750972958283466 events/Boxerbar-750972958283466.json +# scrape_page 153799614679194 events/Britannia-153799614679194.Hotel.json +# scrape_page 380071843764 events/ByscenenTrondheim-380071843764.json +# scrape_page 220501644709429 events/CinemateketTrondheim-220501644709429.json +# scrape_page 225737590803212 events/Diskoteketbar-225737590803212.json +# scrape_page 325557631163470 events/Dora3Trondheim-325557631163470.json +# scrape_page 784700614931033 events/Goodomensbar-784700614931033.json +# scrape_page 1673051982745157 events/HjortenScene-1673051982745157.json +# scrape_page 11541693212 events/ISFiTNorway-11541693212.json +# scrape_page 1419662314995709 events/KafeSkuret-1419662314995709.json +# scrape_page 1092418890774152 events/Kafelarssen-1092418890774152.json +# scrape_page 154134355099614 events/KonsertGalleriet-154134355099614.json +# scrape_page 2198343290262409 events/Lager11-2198343290262409.trondheim.json +# scrape_page 287716286232 events/Pstereo-287716286232.json +# scrape_page 111985600537696 events/Sommer-i-Borggården-111985600537696-111985600537696.json +# scrape_page 151868258198492 events/Sukkerhuset-151868258198492.json +# scrape_page 274218068628 events/TRONDHEIMISAK-274218068628.json +# scrape_page 705753499633189 events/TrondheimCamping-705753499633189.json +# scrape_page 214229302015789 events/TrondheimJazzforum-214229302015789.json +# scrape_page 311676952866578 events/TrondheimLive-311676952866578.json +# scrape_page 114647705270950 events/TrondheimMetalFest-114647705270950.json +# scrape_page 1866856830009864 events/TrondheimRocks-1866856830009864.json +# scrape_page 175439175828277 events/TrondheimSpektrum-175439175828277.json +# scrape_page 108747454194270 events/TrykkerietScene-108747454194270.json +# scrape_page 626731467352429 events/WorkWorkTrd-626731467352429.json +# scrape_page 300444313497473 events/almasrestaurant-300444313497473.json +# scrape_page 106346922773023 events/antikvarene-106346922773023.json +# scrape_page 150118755488 events/artscenetrondheim-150118755488.json +# scrape_page 158501780863403 events/barcircus-158501780863403.json +# scrape_page 160806787399376 events/barmoskus-160806787399376.json +# scrape_page 1712859978926496 events/brorbaras-1712859978926496.json +# scrape_page 191072824261732 events/cafenim-191072824261732.json +# scrape_page 129738423764486 events/cafenimuser-129738423764486.json +# scrape_page 118990964789505 events/clarioncollectiongrandolav-118990964789505.json +# scrape_page 148274751888665 events/clariontrondheim-148274751888665.json +# scrape_page 378038755927834 events/comforthpark-378038755927834.json +# scrape_page 271908336649895 events/crispingloverrecordshop-271908336649895.json +# scrape_page 860188704010725 events/feminalen-860188704010725.json +# scrape_page 654277341296114 events/fru-654277341296114.lundgreen.json +# scrape_page 955982084466587 events/habitattrondheim-955982084466587.json +# scrape_page 143110949094001 events/ilabrainnstasjon-143110949094001.json +# scrape_page 105916402763134 events/jazzfestno-105916402763134.json +# scrape_page 80792973478 events/kamfest-80792973478.json +# scrape_page 137220573544106 events/komikerliv-137220573544106.json +# scrape_page 1402925659778914 events/konsertkollektiv-1402925659778914.json +# scrape_page 642243462457769 events/kulturnattrondheim-642243462457769.json +# scrape_page 246603018828671 events/kunsthalltrondheim-246603018828671.json +# scrape_page 180524565650597 events/ladekaia-180524565650597.json +# scrape_page 1552366408164455 events/lamon-1552366408164455.scenen.json +# scrape_page 101846779891549 events/lobbyen-101846779891549.json +# scrape_page 191245541651071 events/lokal-191245541651071.trhm.json +# scrape_page 346065342451673 events/midtbyenrockeklubb-346065342451673.json +# scrape_page 1786002618299415 events/nidelvenbarogscene-1786002618299415.json +# scrape_page 90016673206 events/olavsfest-90016673206.json +# scrape_page 149127815110411 events/olavshallenas-149127815110411.json +# scrape_page 130558850296573 events/qualityhotelpanorama-130558850296573.json +# scrape_page 286956505871 events/ringvemusikkmuseum-286956505871.json +# scrape_page 68623573578 events/rockheim-68623573578.json +# scrape_page 11782406887 events/samfundet-11782406887.json +# scrape_page 375949715790210 events/scandicbakklandet-375949715790210.json +# scrape_page 260116090806604 events/scandiclerkendal-260116090806604.json +# scrape_page 127377617334855 events/scandicnidelven-127377617334855.json +# scrape_page 799107733481572 events/scandicsolsiden-799107733481572.json +# scrape_page 169539553659073 events/scenekanten-169539553659073.trondheim.json +# scrape_page 110148939061528 events/solsidentrondheim-110148939061528.json +# scrape_page 109571644107406 events/stormfestivalen-109571644107406.json +# scrape_page 484033441709379 events/stundomrecords-484033441709379.json +# scrape_page 208801362485221 events/thonhotelprinsen-208801362485221.json +# scrape_page 1649281185339329 events/trollrestaurant-1649281185339329.json +# scrape_page 33687748452 events/trondercore-33687748452.json +# scrape_page 112475092130212 events/trondheim-112475092130212.kunstmuseum.json +# scrape_page 192236567852623 events/trondheimbluesklubb-192236567852623.json +# scrape_page 125977737456625 events/trondheimcalling-125977737456625.json +# scrape_page 16582146804 events/trondheimfolkebibliotek-16582146804.json +# scrape_page 1858914204185866 events/trondheimstage-1858914204185866.json +# scrape_page 237410323543222 events/tyventrondheim-237410323543222.json +# scrape_page 697776233588499 events/uffabookingcrew-697776233588499.json +scrape_page 965958530110723 events/vaarfruekirke-965958530110723.json diff --git a/bin/scrape b/bin/scrape index da62237..da2b682 100755 --- a/bin/scrape +++ b/bin/scrape @@ -14,8 +14,8 @@ if (number == null) { try { const res = await get_page_events({ pageID: number, - get_upcoming_events: false, - get_past_events: true + get_upcoming_events: true, + get_past_events: false }); console.log(JSON.stringify(res)); } catch (e) { diff --git a/src/facebook/get-page-events.mjs b/src/facebook/get-page-events.mjs index 15bc1c0..2920781 100644 --- a/src/facebook/get-page-events.mjs +++ b/src/facebook/get-page-events.mjs @@ -35,12 +35,41 @@ export const upcoming_pagination_query = async ({ pageID, cursor }) => { return page; }; +// PageEventsTabRecurringEventsCardRendererQuery +export const upcoming_reoccuring_render_query = async ({ pageID }) => { + const doc_id = '4458618790924188'; + const resp = await do_request(doc_id, { pageID }); + const page = resp?.data?.page?.upcomingRecurringEvents?.edges ?? null; + + if (page === null || !Array.isArray(page)) { + return null; + } + + let res = []; + + page.forEach(({ node }) => { + let { childEvents, ...parent } = node; + childEvents.edges.forEach(({ node }) => { + let event = { + ...parent, + ...{ + id: node.id, + time_range: { start: new Date(node.currentStartTimestamp * 1000) } + } + }; + res.push(event); + }); + }); + + return res; +}; + export const get_page_events = async ({ pageID, get_past_events, get_upcoming_events }) => { - let past_events = []; + let res = []; if (get_past_events) { const result = await past_render_query({ pageID }); @@ -68,16 +97,13 @@ export const get_page_events = async ({ has_next_page = paginationResult?.page_info?.has_next_page ?? false; cursor = paginationResult.page_info.end_cursor; } - past_events = [...edges]; + res = [...edges.map(({ node }) => node)]; } } - let upcoming_events = []; - if (get_upcoming_events) { const result = await upcoming_render_query({ pageID }); if (result !== null) { - upcoming_events = []; let { has_next_page, end_cursor: cursor } = result.page_info; let { edges } = result; let retries = 0; @@ -98,11 +124,16 @@ export const get_page_events = async ({ has_next_page = paginationResult?.page_info?.has_next_page ?? false; cursor = paginationResult?.page_info?.end_cursor; } - upcoming_events = [...edges]; + + res = [...edges.map(({ node }) => node)]; + } + const reoccuring = await upcoming_reoccuring_render_query({ pageID }); + if (reoccuring !== null) { + res = [...res, ...reoccuring]; } } - return [...upcoming_events, ...past_events].map(({ node }) => node); + return res; }; export default get_page_events;