Browse Source

sync up

fix-broken-scrape
Jørgen Sverre Lien Sellæg 4 years ago
parent
commit
cf45709cbc
  1. 11
      README.md
  2. 3
      bin/remove-pageID
  3. 10
      bin/remove-profile-picture
  4. 14
      bin/scrape-event
  5. 773
      lokal.nodes.json
  6. 7
      src/#maldito-mapper.js#
  7. 17
      src/facebook-event-payload-parser.js
  8. 21
      src/facebook-event.js
  9. 65
      src/facebook-page-events.js
  10. 54
      src/facebook-payload.txt
  11. 23
      src/facebook-request.js
  12. 2
      src/logic.js
  13. 6
      src/maldito-mapper.js
  14. 2636
      src/output.json
  15. 8
      src/scrape-event.js
  16. 4574
      yarn.lock

11
README.md

@ -1,7 +1,9 @@
# Facebook page events scraper # Facebook page events scraper
Now you can get JSON representation of all events hosted by a Facebook page. Now you can get JSON representation of all events hosted by a Facebook page.
## Disclaimer ## Disclaimer
Please note that terms and conditions apply from Facebook and this script should Please note that terms and conditions apply from Facebook and this script should
only be used if written consent is provided by Facebook. only be used if written consent is provided by Facebook.
@ -14,20 +16,23 @@ yarn
google-chrome / chromium google-chrome / chromium
``` ```
NOTE: You might want to set environment variables to use an existing chrome NOTE: You might want to set environment variables to use an existing chrome
install. (Voluntary) install. (Voluntary)
``` sh
```sh
export PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true export PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true
export PUPPETEER_EXECUTABLE_PATH=/bin/google-chrome export PUPPETEER_EXECUTABLE_PATH=/bin/google-chrome
``` ```
## Usage ## Usage
```sh ```sh
./bin/scrape PlanetRammstein ./bin/scrape PlanetRammstein
./bin/scrape --skip-upcoming-events --past-events PlanetRammstein ./bin/scrape --skip-upcoming-events --past-events PlanetRammstein
``` ```
### Usage with docker ### Usage with docker
```sh ```sh
docker build --tag facebook-page-events-scraper . docker build --tag facebook-page-events-scraper .
docker run \ docker run \

3
bin/remove-pageID

@ -1,7 +1,4 @@
#!/usr/bin/env node #!/usr/bin/env node
const pathOr = require('ramda/src/pathOr');
let input = []; let input = [];
process.stdin.resume(); process.stdin.resume();

10
bin/remove-profile-picture

@ -1,22 +1,14 @@
#!/usr/bin/env node #!/usr/bin/env node
const pathOr = require('ramda/src/pathOr');
let input = []; let input = [];
process.stdin.resume(); process.stdin.resume();
process.stdin.setEncoding('utf8'); process.stdin.setEncoding('utf8');
process.stdin.on('data', (data) => input.push(data));
process.stdin.on('data', (data) => {
input.push(data);
});
process.stdin.on('end', () => { process.stdin.on('end', () => {
const str = input.join(''); const str = input.join('');
const events = JSON.parse(str); const events = JSON.parse(str);
let uniquePages = [];
console.log( console.log(
JSON.stringify(events.map(({ profilePicture, ...event }) => event)), JSON.stringify(events.map(({ profilePicture, ...event }) => event)),
); );

14
bin/scrape-event

@ -0,0 +1,14 @@
#!/usr/bin/env node
let input = [];
process.stdin.resume();
process.stdin.setEncoding('utf8');
process.stdin.on('data', (data) => input.push(data));
process.stdin.on('end', () => {
const str = input.join('');
const events = JSON.parse(str);
console.log(
JSON.stringify(events.map(({ profilePicture, ...event }) => event)),
);
});

773
lokal.nodes.json

@ -0,0 +1,773 @@
[
{
"id": "348882696889506",
"name": "Orions Belte — Lokal Klubb",
"url": "https://www.facebook.com/events/348882696889506/",
"event_creator": {
"__typename": "Page",
"id": "191245541651071",
"pay_to_access_onboarding_info": {
"creator": {
"has_active_claimed_ad_coupon": null,
"pending_ad_coupon_content": null,
"id": "191245541651071_pay_to_access_creator"
},
"id": "UE9FQ3JlYXRvck9uYm9hcmRpbmdJbmZvOjE5MTI0NTU0MTY1MTA3MQ=="
}
},
"day_time_sentence": "FRI, AUG 13 AT 11:00 AM UNK",
"is_canceled": false,
"created_for_group": null,
"available_actors": null,
"rsvp_button_renderer": {
"__typename": "PublicRsvpStyleRenderer",
"event": {
"id": "348882696889506",
"connection_style": "INTERESTED",
"can_viewer_join": false,
"can_viewer_watch": false,
"can_viewer_unwatch": false,
"viewer_watch_status": "UNWATCHED",
"event_connection_data_privacy_scope": null,
"privacy_scope_for_toast": null
},
"__module_operation_EventCometUniversalRSVPButton_event": {
"__dr": "PublicEventCometRSVPButtonRenderer_rsvpStyleRenderer$normalization.graphql"
},
"__module_component_EventCometUniversalRSVPButton_event": {
"__dr": "PublicEventCometRSVPButtonRenderer.react"
}
},
"rsvp_button_group_renderer": {
"__typename": "PublicRsvpStyleRenderer",
"event": {
"id": "348882696889506",
"connection_style": "INTERESTED",
"can_viewer_join": false,
"can_viewer_watch": false,
"viewer_watch_status": "UNWATCHED",
"is_past": false
},
"__module_operation_EventCometUniversalRSVPButtonGroup_event": {
"__dr": "PublicEventCometRSVPButtonGroupRenderer_rsvpStyleRenderer$normalization.graphql"
},
"__module_component_EventCometUniversalRSVPButtonGroup_event": {
"__dr": "PublicEventCometRSVPButtonGroupRenderer.react"
}
},
"privacy_scope_for_toast": null,
"rsvp_style": "PUBLIC_RSVP_STYLE",
"viewer_guest_status": null,
"viewer_watch_status": "UNWATCHED",
"event_kind": "PUBLIC_TYPE",
"can_viewer_invite": false,
"can_page_viewer_invite_as_user": false,
"eventUrl": "https://www.facebook.com/events/348882696889506/",
"can_boost_event_renderer": null,
"can_viewer_see_rsvp_button": false,
"can_viewer_share": false,
"can_viewer_create_repeat_event": false,
"header_action_menu_items": [],
"is_viewer_admin": false,
"if_viewer_can_publish_draft_event": null,
"online_event_setup": null,
"paid_online_event_context": null,
"parent_if_exists_or_self": { "id": "348882696889506" },
"event_for_edit_flow": {
"if_viewer_can_edit": null,
"id": "348882696889506"
},
"is_eligible_for_poe_view_as_visitor_button": false,
"discovery_categories": [{ "glyph_token": "music" }],
"event_connected_users": { "count": 0, "edges": [] },
"social_context": { "text": "33 people interested" },
"__typename": "Event",
"ticketUrl": "http://www.tikkio.com/lokalbar",
"hosts": {
"edges": [
{
"node": {
"__typename": "Page",
"name": "Polar Artist",
"pageID": "121444301228925",
"about": {
"text": "International music management and live booking agency",
"ranges": []
},
"category": "Künstlervermittlung",
"city": null,
"profilePicture": {
"uri": "https://scontent-vie1-1.xx.fbcdn.net/v/t1.18169-1/p100x100/13226716_1192511330788878_2675807659535641780_n.png?_nc_cat=108&ccb=1-3&_nc_sid=1eb0c7&_nc_ohc=hekFCEl8ChsAX_1TFQ7&_nc_ht=scontent-vie1-1.xx&oh=5a1621b48159069d414ba2ee32f438df&oe=612BC18F"
},
"url": "https://www.facebook.com/polarartist/",
"id": "121444301228925"
}
},
{
"node": {
"__typename": "Page",
"name": "Lokal Bar - Scene - Klubb",
"pageID": "191245541651071",
"about": { "text": "——— FRIROMMET I DIN BY ———", "ranges": [] },
"category": "Bar",
"city": { "name": "Trondheim", "id": "110429825645017" },
"profilePicture": {
"uri": "https://scontent-vie1-1.xx.fbcdn.net/v/t1.6435-1/p100x100/68638643_486313585477597_4139052235034198016_n.png?_nc_cat=106&ccb=1-3&_nc_sid=1eb0c7&_nc_ohc=yIL4_Rogc4cAX9j8Fwb&_nc_ht=scontent-vie1-1.xx&oh=f5e126836f7592dee1b6441ce38b58ab&oe=61293641"
},
"url": "https://www.facebook.com/lokal.trhm/",
"id": "191245541651071"
}
}
]
},
"place": {
"__typename": "Page",
"name": "Lokal Bar - Scene - Klubb",
"pageID": "191245541651071",
"about": { "text": "——— FRIROMMET I DIN BY ———", "ranges": [] },
"category": "Bar",
"city": { "name": "Trondheim", "id": "110429825645017" },
"profilePicture": {
"uri": "https://scontent-vie1-1.xx.fbcdn.net/v/t1.6435-1/p100x100/68638643_486313585477597_4139052235034198016_n.png?_nc_cat=106&ccb=1-3&_nc_sid=1eb0c7&_nc_ohc=yIL4_Rogc4cAX9j8Fwb&_nc_ht=scontent-vie1-1.xx&oh=f5e126836f7592dee1b6441ce38b58ab&oe=61293641"
},
"url": "https://www.facebook.com/lokal.trhm/",
"id": "191245541651071"
},
"is_arena_event": false,
"unified_tournament": null,
"arenas": { "nodes": [] },
"remote_learning_class": null,
"fundraiser_charity": null,
"tz_display_name": "UTC+02",
"viewer_in_event_tz": false,
"start_timestamp": 1628877600,
"end_timestamp": 1628888400,
"linked_group_card": null,
"event_linked_group_link_action_data": {
"linkable_groups": { "is_empty": null }
},
"live_virtual_event_info": null,
"liveVideos": null,
"event_to_event_tours": { "nodes": [] },
"aboutTabTools": null,
"paidAccessTabTools": null,
"event_description": {
"text": "— dørene åpner 20:00\n— konsertstart 21:00\n— id 20\n\n🎫 billetter i salg fra mandag 12. august\n😷 lokal.bar/smittevern \n\n——\n\nDet var duket for slippturné rundt \"Villa Amorini\" tidligere i år, men dessverre spøkte det for konserten i Trondheim pga smitteoppblussing. Nå har vi ny dato klar på den andre siden av sommeren, og Orions Belte har i tillegg gitt ut den fantastiske live-skiva \"The Scenic Route!\"\n\n——\n\nHusker du den gangen dørvakta løp etter de fulle ungdommene langs kaia? Når han stuper etter dem uti vannet skraper han seg opp på magen, men han tar dem igjen til slutt og returnerer til klubben med en våt villstyring under hver arm. Samtidig fortsetter bandet å spille inne på klubben, og vokalisten står på galleriet og håper at lysekrona holder i det han hopper mot den. Det gjør den ikke. Husker du nachspielene som aldri tok slutt, og den evige kampen om å unngå besøk av politiet og skjenkekontrollen? Nei? Ingenting?\nDette var klubbscenen på nittitallet i Bergen, og Villa Amorini var stedet der alt skjedde.\n\n——"
},
"share_url": "https://facebook.com/events/s/orions-belte-lokal-klubb/348882696889506/",
"is_eligible_for_poe_visit_dashboard_card": false,
"event_if_remote_learning_course_classes": null,
"associated_videos": null,
"is_online": false,
"has_learning_space_thread": false,
"has_messenger_room": false,
"has_fb_live": false,
"has_third_party": false,
"faq_items": { "nodes": [] },
"can_viewer_see_nit_card": false,
"nit_card_title": null,
"nit_card_body": null,
"nit_card_button_text": null,
"nit_card_uri": null,
"event_lineups": {
"edges": [],
"page_info": { "end_cursor": null, "has_next_page": false }
},
"event_notable_sources": { "count": 0, "nodes": [] },
"suggested_event_cuts_of_event": {
"nodes": [
{
"cut_type": "FRIENDS_EVENTS",
"displayed_text": "Populair bij vrienden",
"events": { "count": 0, "nodes": [] }
}
]
},
"can_viewer_message": false,
"ticket_settings": { "nodes": [] },
"event_hosts_that_can_view_guestlist": [
{
"__typename": "Page",
"id": "191245541651071",
"viewer": { "actor": { "__typename": "LoggedOutUser", "id": "0" } },
"url": "https://www.facebook.com/lokal.trhm/",
"name": "Lokal Bar - Scene - Klubb",
"past_event_count": { "count": 405 },
"page_about_fields": { "blurb": "——— FRIROMMET I DIN BY ———" },
"page_likers": { "global_likers_count": 3670 },
"profile_uri": {
"uri": "https://scontent-amt2-1.xx.fbcdn.net/v/t1.6435-1/p200x200/68638643_486313585477597_4139052235034198016_n.png?_nc_cat=106&ccb=1-3&_nc_sid=1eb0c7&_nc_ohc=yIL4_Rogc4cAX8Xtuqr&_nc_ht=scontent-amt2-1.xx&oh=a0f6b1025b961c07084cb88196193892&oe=612B1A30"
},
"cover_url": {
"photo": {
"image": {
"uri": "https://scontent-amt2-1.xx.fbcdn.net/v/t31.18172-8/s960x960/30425394_210293716412920_4586078545182596007_o.png?_nc_cat=109&ccb=1-3&_nc_sid=e3f864&_nc_ohc=eY6iGKrBrx4AX-Ja9PW&_nc_oc=AQnLkP6ysKuxZ1ilhjCb59LwXq0oZvA1JUvCqMfueuet7tIVi35Qg8GgeD_dgog_yqA&_nc_ht=scontent-amt2-1.xx&oh=1076b4497c15a5777d68ab303f6f3bd4&oe=612AEEF1"
},
"id": "210293716412920"
}
},
"is_viewer_fan": false,
"can_viewer_like": false,
"should_show_page_like_warning": false
},
{
"__typename": "Page",
"id": "121444301228925",
"viewer": { "actor": { "__typename": "LoggedOutUser", "id": "0" } },
"url": "https://www.facebook.com/polarartist/",
"name": "Polar Artist",
"past_event_count": { "count": 560 },
"page_about_fields": {
"blurb": "International music management and live booking agency"
},
"page_likers": { "global_likers_count": 2778 },
"profile_uri": {
"uri": "https://scontent-ams4-1.xx.fbcdn.net/v/t1.18169-1/p200x200/13226716_1192511330788878_2675807659535641780_n.png?_nc_cat=108&ccb=1-3&_nc_sid=1eb0c7&_nc_ohc=hekFCEl8ChsAX9ZTm2M&_nc_ht=scontent-ams4-1.xx&oh=dd5f134c856f0a48d5a7fd494fd489b2&oe=6128F92E"
},
"cover_url": {
"photo": {
"image": {
"uri": "https://scontent-ams4-1.xx.fbcdn.net/v/t1.6435-9/s960x960/170848750_5706897106016922_6014426456866089959_n.jpg?_nc_cat=108&ccb=1-3&_nc_sid=e3f864&_nc_ohc=P3Gc1aa6buIAX9ojrI8&_nc_oc=AQnMaCiqlBMbOUzmrOGtylpCTmegia6A-_otZtuiAr0h3zXfuYOTQyRZSbx2EHtUwyg&_nc_ht=scontent-ams4-1.xx&oh=a29b89a03f3f0eedc5e3fa9ec15889a5&oe=612A8003"
},
"id": "5706897096016923"
}
},
"is_viewer_fan": false,
"can_viewer_like": false,
"should_show_page_like_warning": false
}
],
"can_viewer_see_transparency_card": true,
"transparency": {
"name_change_history_info": { "has_name_changed": false },
"date_change_history_info": { "has_date_changed": false }
}
},
{
"id": "405702597560924",
"name": "Linni — Lokal Klubb",
"url": "https://www.facebook.com/events/405702597560924/",
"event_creator": {
"__typename": "Page",
"id": "191245541651071",
"pay_to_access_onboarding_info": {
"creator": {
"has_active_claimed_ad_coupon": null,
"pending_ad_coupon_content": null,
"id": "191245541651071_pay_to_access_creator"
},
"id": "UE9FQ3JlYXRvck9uYm9hcmRpbmdJbmZvOjE5MTI0NTU0MTY1MTA3MQ=="
}
},
"day_time_sentence": "SAT, AUG 28 AT 10:30 AM UNK",
"is_canceled": false,
"created_for_group": null,
"available_actors": null,
"rsvp_button_renderer": {
"__typename": "PublicRsvpStyleRenderer",
"event": {
"id": "405702597560924",
"connection_style": "INTERESTED",
"can_viewer_join": false,
"can_viewer_watch": false,
"can_viewer_unwatch": false,
"viewer_watch_status": "UNWATCHED",
"event_connection_data_privacy_scope": null,
"privacy_scope_for_toast": null
},
"__module_operation_EventCometUniversalRSVPButton_event": {
"__dr": "PublicEventCometRSVPButtonRenderer_rsvpStyleRenderer$normalization.graphql"
},
"__module_component_EventCometUniversalRSVPButton_event": {
"__dr": "PublicEventCometRSVPButtonRenderer.react"
}
},
"rsvp_button_group_renderer": {
"__typename": "PublicRsvpStyleRenderer",
"event": {
"id": "405702597560924",
"connection_style": "INTERESTED",
"can_viewer_join": false,
"can_viewer_watch": false,
"viewer_watch_status": "UNWATCHED",
"is_past": false
},
"__module_operation_EventCometUniversalRSVPButtonGroup_event": {
"__dr": "PublicEventCometRSVPButtonGroupRenderer_rsvpStyleRenderer$normalization.graphql"
},
"__module_component_EventCometUniversalRSVPButtonGroup_event": {
"__dr": "PublicEventCometRSVPButtonGroupRenderer.react"
}
},
"privacy_scope_for_toast": null,
"rsvp_style": "PUBLIC_RSVP_STYLE",
"viewer_guest_status": null,
"viewer_watch_status": "UNWATCHED",
"event_kind": "PUBLIC_TYPE",
"can_viewer_invite": false,
"can_page_viewer_invite_as_user": false,
"eventUrl": "https://www.facebook.com/events/405702597560924/",
"can_boost_event_renderer": null,
"can_viewer_see_rsvp_button": false,
"can_viewer_share": false,
"can_viewer_create_repeat_event": false,
"header_action_menu_items": [],
"is_viewer_admin": false,
"if_viewer_can_publish_draft_event": null,
"online_event_setup": null,
"paid_online_event_context": null,
"parent_if_exists_or_self": { "id": "405702597560924" },
"event_for_edit_flow": {
"if_viewer_can_edit": null,
"id": "405702597560924"
},
"is_eligible_for_poe_view_as_visitor_button": false,
"discovery_categories": [],
"event_connected_users": { "count": 0, "edges": [] },
"social_context": { "text": "Music · 144 people" },
"__typename": "Event",
"ticketUrl": "http://www.tikkio.com/lokalbar",
"hosts": {
"edges": [
{
"node": {
"__typename": "Page",
"name": "Linje2",
"pageID": "1022549294549528",
"about": { "text": "", "ranges": [] },
"category": "Concert Tour",
"city": null,
"profilePicture": {
"uri": "https://scontent.frix7-1.fna.fbcdn.net/v/t1.18169-1/p100x100/27657023_1037770509694073_6986473411550183445_n.jpg?_nc_cat=104&ccb=1-3&_nc_sid=1eb0c7&_nc_ohc=NVayHfJoB0QAX9Sb2F2&_nc_ht=scontent.frix7-1.fna&oh=430f746f19f54a1e4f016ee82e9fcd81&oe=612933BB"
},
"url": "https://www.facebook.com/linje2booking/",
"id": "1022549294549528"
}
},
{
"node": {
"__typename": "Page",
"name": "Linni",
"pageID": "801499590006231",
"about": {
"text": "Levemann i lekeland, rapper, danser, elsker, flammegutt, neonsurfer & eventyrer. Sjekk ut albumene Minimum & Maksimum mens du venter på Medium.",
"ranges": []
},
"category": "Artist",
"city": null,
"profilePicture": {
"uri": "https://scontent.frix7-1.fna.fbcdn.net/v/t1.6435-1/p100x100/187990303_2015135801975931_8549619315634067253_n.jpg?_nc_cat=111&ccb=1-3&_nc_sid=1eb0c7&_nc_ohc=snsN8dQm8vsAX9QzJ_B&_nc_ht=scontent.frix7-1.fna&oh=093b4d791ff2e7d91d7bdf493ba0a9bb&oe=612B9710"
},
"url": "https://www.facebook.com/ygmglinni/",
"id": "801499590006231"
}
},
{
"node": {
"__typename": "Page",
"name": "Lokal Bar - Scene - Klubb",
"pageID": "191245541651071",
"about": { "text": "——— FRIROMMET I DIN BY ———", "ranges": [] },
"category": "Bar",
"city": { "name": "Trondheim, Norway", "id": "110429825645017" },
"profilePicture": {
"uri": "https://scontent.frix7-1.fna.fbcdn.net/v/t1.6435-1/p100x100/68638643_486313585477597_4139052235034198016_n.png?_nc_cat=106&ccb=1-3&_nc_sid=1eb0c7&_nc_ohc=yIL4_Rogc4cAX-LWpHb&_nc_ht=scontent.frix7-1.fna&oh=273b8e61292048bb416b9eb814a5875b&oe=61293641"
},
"url": "https://www.facebook.com/lokal.trhm/",
"id": "191245541651071"
}
}
]
},
"place": {
"__typename": "Page",
"name": "Lokal Bar - Scene - Klubb",
"pageID": "191245541651071",
"about": { "text": "——— FRIROMMET I DIN BY ———", "ranges": [] },
"category": "Bar",
"city": { "name": "Trondheim, Norway", "id": "110429825645017" },
"profilePicture": {
"uri": "https://scontent.frix7-1.fna.fbcdn.net/v/t1.6435-1/p100x100/68638643_486313585477597_4139052235034198016_n.png?_nc_cat=106&ccb=1-3&_nc_sid=1eb0c7&_nc_ohc=yIL4_Rogc4cAX-LWpHb&_nc_ht=scontent.frix7-1.fna&oh=273b8e61292048bb416b9eb814a5875b&oe=61293641"
},
"url": "https://www.facebook.com/lokal.trhm/",
"id": "191245541651071"
},
"is_arena_event": false,
"unified_tournament": null,
"arenas": { "nodes": [] },
"remote_learning_class": null,
"fundraiser_charity": null,
"tz_display_name": "UTC+02",
"viewer_in_event_tz": false,
"start_timestamp": 1630171800,
"end_timestamp": 1630186200,
"linked_group_card": null,
"event_linked_group_link_action_data": {
"linkable_groups": { "is_empty": null }
},
"live_virtual_event_info": null,
"liveVideos": null,
"event_to_event_tours": { "nodes": [] },
"aboutTabTools": null,
"paidAccessTabTools": null,
"event_description": {
"text": "— to konserter, kl 20 og kl 22\n— dørene åpner 30 minutter tidligere\n— id 20\n\n🎫 billetter i salg fra mandag 19. august\n😷 lokal.bar/smittevern\n\n——\n\nJonas Grieg aka LINNI er en av de mest produktive artistene vi har sett her til lands. Etter 'Minimum' / 'Medium' / 'Maksimum' trilogien, 'Silkesvarten', sideprosjektet Neste Planet og nye låter med Yoguttene har han i år gitt ut fantastiske 'Saga'... men det stopper ikke der.\n\nRett over sommeren kommer 'Nightridah', et tech/house inspirert album som skapt for å sette fyr på klubbgulv.\n\nFørstesingelen 'Barakuda' gir masse mersmak, vi har lenge hatt lyst til å invitere Linni til Lokal, og nå er timingen perfekt.\n\nBEGRENSET KAPASITET på 50 stoler per konsert, derfor setter vi opp to omganger kl 19 og kl 21.\n\nSkulle det bli lettelser på arrangementsføringer innen den tid så åpner vi gjerne opp for mer.\n\n——"
},
"share_url": "https://facebook.com/events/s/linni-lokal-klubb/405702597560924/",
"is_eligible_for_poe_visit_dashboard_card": false,
"event_if_remote_learning_course_classes": null,
"associated_videos": null,
"is_online": false,
"has_learning_space_thread": false,
"has_messenger_room": false,
"has_fb_live": false,
"has_third_party": false,
"can_viewer_see_nit_card": false,
"nit_card_title": null,
"nit_card_body": null,
"nit_card_button_text": null,
"nit_card_uri": null,
"event_lineups": {
"edges": [],
"page_info": { "end_cursor": null, "has_next_page": false }
},
"faq_items": { "nodes": [] },
"event_notable_sources": { "count": 0, "nodes": [] },
"can_viewer_see_transparency_card": true,
"transparency": {
"name_change_history_info": { "has_name_changed": false },
"date_change_history_info": { "has_date_changed": false }
},
"suggested_event_cuts_of_event": {
"nodes": [
{
"cut_type": "FRIENDS_EVENTS",
"displayed_text": "Popular With Friends",
"events": { "count": 0, "nodes": [] }
}
]
},
"can_viewer_message": false,
"ticket_settings": { "nodes": [] },
"event_hosts_that_can_view_guestlist": [
{
"__typename": "Page",
"id": "1022549294549528",
"viewer": { "actor": { "__typename": "LoggedOutUser", "id": "0" } },
"url": "https://www.facebook.com/linje2booking/",
"name": "Linje2",
"past_event_count": { "count": 42 },
"page_about_fields": { "blurb": null },
"page_likers": { "global_likers_count": 581 },
"profile_uri": {
"uri": "https://scontent.frix7-1.fna.fbcdn.net/v/t1.18169-1/p200x200/27657023_1037770509694073_6986473411550183445_n.jpg?_nc_cat=104&ccb=1-3&_nc_sid=1eb0c7&_nc_ohc=NVayHfJoB0QAX-Yv-OS&_nc_ht=scontent.frix7-1.fna&oh=fa10477e5711ecbf738347536805c981&oe=6128FB1A"
},
"cover_url": {
"photo": {
"image": {
"uri": "https://scontent.frix7-1.fna.fbcdn.net/v/t1.6435-9/51655236_1320357664768688_6674026990665728000_n.png?_nc_cat=108&ccb=1-3&_nc_sid=e3f864&_nc_ohc=41HI-euomVsAX-1S9Z7&_nc_ht=scontent.frix7-1.fna&oh=81ec56779b118f153f9ad7e6964bd4b9&oe=612B17E7"
},
"id": "1320357658102022"
}
},
"is_viewer_fan": false,
"can_viewer_like": false,
"should_show_page_like_warning": false
},
{
"__typename": "Page",
"id": "801499590006231",
"viewer": { "actor": { "__typename": "LoggedOutUser", "id": "0" } },
"url": "https://www.facebook.com/ygmglinni/",
"name": "Linni",
"past_event_count": { "count": 15 },
"page_about_fields": {
"blurb": "Levemann i lekeland, rapper, danser, elsker, flammegutt, neonsurfer & eventyrer. Sjekk ut albumene Minimum & Maksimum mens du venter på Medium."
},
"page_likers": { "global_likers_count": 954 },
"profile_uri": {
"uri": "https://scontent.frix7-1.fna.fbcdn.net/v/t1.6435-1/p200x200/187990303_2015135801975931_8549619315634067253_n.jpg?_nc_cat=111&ccb=1-3&_nc_sid=1eb0c7&_nc_ohc=snsN8dQm8vsAX9-cdBk&_nc_ht=scontent.frix7-1.fna&oh=d3659ca1a9fc4f4c825c96c9a2af6356&oe=612BCB27"
},
"cover_url": {
"photo": {
"image": {
"uri": "https://scontent.frix7-1.fna.fbcdn.net/v/t1.6435-9/s960x960/188465145_2015133865309458_4721806180284182920_n.jpg?_nc_cat=105&ccb=1-3&_nc_sid=e3f864&_nc_ohc=GQXjmMizm8gAX9ecKWM&_nc_ht=scontent.frix7-1.fna&oh=acbae08d2620d08026593bfdb4437f2e&oe=6128E619"
},
"id": "2015133861976125"
}
},
"is_viewer_fan": false,
"can_viewer_like": false,
"should_show_page_like_warning": false
},
{
"__typename": "Page",
"id": "191245541651071",
"viewer": { "actor": { "__typename": "LoggedOutUser", "id": "0" } },
"url": "https://www.facebook.com/lokal.trhm/",
"name": "Lokal Bar - Scene - Klubb",
"past_event_count": { "count": 405 },
"page_about_fields": { "blurb": "——— FRIROMMET I DIN BY ———" },
"page_likers": { "global_likers_count": 3670 },
"profile_uri": {
"uri": "https://scontent.frix7-1.fna.fbcdn.net/v/t1.6435-1/p200x200/68638643_486313585477597_4139052235034198016_n.png?_nc_cat=106&ccb=1-3&_nc_sid=1eb0c7&_nc_ohc=yIL4_Rogc4cAX_DUOf6&_nc_ht=scontent.frix7-1.fna&oh=fed660991a9130d02caa7073b32adcaf&oe=612B1A30"
},
"cover_url": {
"photo": {
"image": {
"uri": "https://scontent.frix7-1.fna.fbcdn.net/v/t31.18172-8/s960x960/30425394_210293716412920_4586078545182596007_o.png?_nc_cat=109&ccb=1-3&_nc_sid=e3f864&_nc_ohc=eY6iGKrBrx4AX_wtK4N&_nc_oc=AQkBdMEEkgps95apQEW4AxiYdXj1SHiOQaw22FV2ER3yicGG1TrKGetG83r4wOKSfiA&_nc_ht=scontent.frix7-1.fna&oh=34c8d6afddc2914e595b53d38297a02d&oe=612AEEF1"
},
"id": "210293716412920"
}
},
"is_viewer_fan": false,
"can_viewer_like": false,
"should_show_page_like_warning": false
}
]
},
{
"id": "760253917947304",
"name": "Hilma Nikolaisen — Lokal Klubb",
"url": "https://www.facebook.com/events/760253917947304/",
"event_creator": {
"__typename": "Page",
"id": "191245541651071",
"pay_to_access_onboarding_info": {
"creator": {
"has_active_claimed_ad_coupon": null,
"pending_ad_coupon_content": null,
"id": "191245541651071_pay_to_access_creator"
},
"id": "UE9FQ3JlYXRvck9uYm9hcmRpbmdJbmZvOjE5MTI0NTU0MTY1MTA3MQ=="
}
},
"day_time_sentence": "FRI, SEP 17 AT 11:00 AM UNK",
"is_canceled": false,
"created_for_group": null,
"available_actors": null,
"rsvp_button_renderer": {
"__typename": "PublicRsvpStyleRenderer",
"event": {
"id": "760253917947304",
"connection_style": "INTERESTED",
"can_viewer_join": false,
"can_viewer_watch": false,
"can_viewer_unwatch": false,
"viewer_watch_status": "UNWATCHED",
"event_connection_data_privacy_scope": null,
"privacy_scope_for_toast": null
},
"__module_operation_EventCometUniversalRSVPButton_event": {
"__dr": "PublicEventCometRSVPButtonRenderer_rsvpStyleRenderer$normalization.graphql"
},
"__module_component_EventCometUniversalRSVPButton_event": {
"__dr": "PublicEventCometRSVPButtonRenderer.react"
}
},
"rsvp_button_group_renderer": {
"__typename": "PublicRsvpStyleRenderer",
"event": {
"id": "760253917947304",
"connection_style": "INTERESTED",
"can_viewer_join": false,
"can_viewer_watch": false,
"viewer_watch_status": "UNWATCHED",
"is_past": false
},
"__module_operation_EventCometUniversalRSVPButtonGroup_event": {
"__dr": "PublicEventCometRSVPButtonGroupRenderer_rsvpStyleRenderer$normalization.graphql"
},
"__module_component_EventCometUniversalRSVPButtonGroup_event": {
"__dr": "PublicEventCometRSVPButtonGroupRenderer.react"
}
},
"privacy_scope_for_toast": null,
"rsvp_style": "PUBLIC_RSVP_STYLE",
"viewer_guest_status": null,
"viewer_watch_status": "UNWATCHED",
"event_kind": "PUBLIC_TYPE",
"can_viewer_invite": false,
"can_page_viewer_invite_as_user": false,
"eventUrl": "https://www.facebook.com/events/760253917947304/",
"can_boost_event_renderer": null,
"can_viewer_see_rsvp_button": false,
"can_viewer_share": false,
"can_viewer_create_repeat_event": false,
"header_action_menu_items": [],
"is_viewer_admin": false,
"if_viewer_can_publish_draft_event": null,
"online_event_setup": null,
"paid_online_event_context": null,
"parent_if_exists_or_self": { "id": "760253917947304" },
"event_for_edit_flow": {
"if_viewer_can_edit": null,
"id": "760253917947304"
},
"is_eligible_for_poe_view_as_visitor_button": false,
"discovery_categories": [{ "glyph_token": "music" }],
"event_connected_users": { "count": 0, "edges": [] },
"social_context": { "text": "45 people interested" },
"__typename": "Event",
"ticketUrl": "http://billetter.lokal.bar/tikkio",
"hosts": {
"edges": [
{
"node": {
"__typename": "Page",
"name": "Polar Artist",
"pageID": "121444301228925",
"about": {
"text": "International music management and live booking agency",
"ranges": []
},
"category": "Agent artistique",
"city": null,
"profilePicture": {
"uri": "https://scontent-frt3-1.xx.fbcdn.net/v/t1.18169-1/p100x100/13226716_1192511330788878_2675807659535641780_n.png?_nc_cat=108&ccb=1-3&_nc_sid=1eb0c7&_nc_ohc=hekFCEl8ChsAX-tRWR-&_nc_ht=scontent-frt3-1.xx&oh=1057f7937afa7f43344706b906c395e9&oe=612BC18F"
},
"url": "https://www.facebook.com/polarartist/",
"id": "121444301228925"
}
},
{
"node": {
"__typename": "Page",
"name": "Lokal Bar - Scene - Klubb",
"pageID": "191245541651071",
"about": { "text": "——— FRIROMMET I DIN BY ———", "ranges": [] },
"category": "Bar",
"city": { "name": "Trondheim", "id": "110429825645017" },
"profilePicture": {
"uri": "https://scontent-frt3-1.xx.fbcdn.net/v/t1.6435-1/p100x100/68638643_486313585477597_4139052235034198016_n.png?_nc_cat=106&ccb=1-3&_nc_sid=1eb0c7&_nc_ohc=yIL4_Rogc4cAX-QElEm&_nc_ht=scontent-frt3-1.xx&oh=53baa2ad424aac815f700522c1c462d3&oe=61293641"
},
"url": "https://www.facebook.com/lokal.trhm/",
"id": "191245541651071"
}
}
]
},
"place": {
"__typename": "Page",
"name": "Lokal Bar - Scene - Klubb",
"pageID": "191245541651071",
"about": { "text": "——— FRIROMMET I DIN BY ———", "ranges": [] },
"category": "Bar",
"city": { "name": "Trondheim", "id": "110429825645017" },
"profilePicture": {
"uri": "https://scontent-frt3-1.xx.fbcdn.net/v/t1.6435-1/p100x100/68638643_486313585477597_4139052235034198016_n.png?_nc_cat=106&ccb=1-3&_nc_sid=1eb0c7&_nc_ohc=yIL4_Rogc4cAX-QElEm&_nc_ht=scontent-frt3-1.xx&oh=53baa2ad424aac815f700522c1c462d3&oe=61293641"
},
"url": "https://www.facebook.com/lokal.trhm/",
"id": "191245541651071"
},
"is_arena_event": false,
"unified_tournament": null,
"arenas": { "nodes": [] },
"remote_learning_class": null,
"fundraiser_charity": null,
"tz_display_name": "UTC+02",
"viewer_in_event_tz": false,
"start_timestamp": 1631901600,
"end_timestamp": 0,
"linked_group_card": null,
"event_linked_group_link_action_data": {
"linkable_groups": { "is_empty": null }
},
"live_virtual_event_info": null,
"liveVideos": null,
"event_to_event_tours": { "nodes": [] },
"aboutTabTools": null,
"paidAccessTabTools": null,
"event_description": {
"text": "NY KONSERTDATO ⤳ 17.09.21\nbillettsalg annonseres mot sommeren\ndørene åpner 20:00\nkonsertstart 21:00\nid 18\n\n——\n\nHilma Nikolaisen er tilbake med sitt tredje album «Heritage», som har blitt en nedstrippet men selvsikker og ikke minst livslystig halvtime med herlig musikk. Vi ønsker henne hjertelig velkommen tilbake til Trondheim med denne intime konsertkvelden!\n\nEtter to kritikerroste album kunne Hilma Nikolaisen lett fortsatt i samme retning. Groovy gitarrock bygget på kontante riff og fengende psykedeliske ekskursjoner har hun tross alt perfeksjonert. I stedet har hun med Heritage tatt et overraskende skritt til siden.\n\nDet nye albumets nedstrippede og tilnærmet akustiske lydbilde har vært et ytterst bevisst valg, og ikke et resultat av tidens traurige, pandemiske begrensninger. Resultatet har blitt Nikolaisens mest tydelige og gjennomarbeidede album, og muligens også det mest umiddelbare.\n\nSingelen «Maybe Today (Satan)» åpner albumet og setter anslaget. Nostalgisk fingerspill på en nedarvet Santana nylongitar fra 1975, et kor av tverrfløyter og en melodi som nærmest er bacharachsk i sin utsøkte enkelhet.\n\n\"Hilma Nikolaisens beste soloplate er her.\" – Audun Vinger, DN Magasinet\n\n——"
},
"share_url": "https://facebook.com/events/s/hilma-nikolaisen-lokal-klubb/760253917947304/",
"is_eligible_for_poe_visit_dashboard_card": false,
"event_if_remote_learning_course_classes": null,
"associated_videos": null,
"is_online": false,
"has_learning_space_thread": false,
"has_messenger_room": false,
"has_fb_live": false,
"has_third_party": false,
"can_viewer_see_nit_card": false,
"nit_card_title": null,
"nit_card_body": null,
"nit_card_button_text": null,
"nit_card_uri": null,
"event_notable_sources": { "count": 0, "nodes": [] },
"event_lineups": {
"edges": [],
"page_info": { "end_cursor": null, "has_next_page": false }
},
"faq_items": { "nodes": [] },
"suggested_event_cuts_of_event": {
"nodes": [
{
"cut_type": "FRIENDS_EVENTS",
"displayed_text": "Beliebt bei Freunden",
"events": { "count": 0, "nodes": [] }
}
]
},
"can_viewer_message": false,
"ticket_settings": { "nodes": [] },
"event_hosts_that_can_view_guestlist": [
{
"__typename": "Page",
"id": "191245541651071",
"viewer": { "actor": { "__typename": "LoggedOutUser", "id": "0" } },
"url": "https://www.facebook.com/lokal.trhm/",
"name": "Lokal Bar - Scene - Klubb",
"past_event_count": { "count": 405 },
"page_about_fields": { "blurb": "——— FRIROMMET I DIN BY ———" },
"page_likers": { "global_likers_count": 3670 },
"profile_uri": {
"uri": "https://scontent.frix7-1.fna.fbcdn.net/v/t1.6435-1/p200x200/68638643_486313585477597_4139052235034198016_n.png?_nc_cat=106&ccb=1-3&_nc_sid=1eb0c7&_nc_ohc=yIL4_Rogc4cAX-n-CJE&_nc_ht=scontent.frix7-1.fna&oh=723fc198b1c979dbfbd8c68cafeab2ee&oe=612B1A30"
},
"cover_url": {
"photo": {
"image": {
"uri": "https://scontent.frix7-1.fna.fbcdn.net/v/t31.18172-8/s960x960/30425394_210293716412920_4586078545182596007_o.png?_nc_cat=109&ccb=1-3&_nc_sid=e3f864&_nc_ohc=eY6iGKrBrx4AX8KjQUJ&_nc_oc=AQmahJMBcGqH2Z4ZlhYoYvjZ2tVF2xGwPPJBPucmPXwxM_hW3GzRk16qHjxtCDZPrG4&_nc_ht=scontent.frix7-1.fna&oh=9e83fac8ea96f0ea0c62f9999e8a51d0&oe=612AEEF1"
},
"id": "210293716412920"
}
},
"is_viewer_fan": false,
"can_viewer_like": false,
"should_show_page_like_warning": false
},
{
"__typename": "Page",
"id": "121444301228925",
"viewer": { "actor": { "__typename": "LoggedOutUser", "id": "0" } },
"url": "https://www.facebook.com/polarartist/",
"name": "Polar Artist",
"past_event_count": { "count": 560 },
"page_about_fields": {
"blurb": "International music management and live booking agency"
},
"page_likers": { "global_likers_count": 2778 },
"profile_uri": {
"uri": "https://scontent.frix7-1.fna.fbcdn.net/v/t1.18169-1/p200x200/13226716_1192511330788878_2675807659535641780_n.png?_nc_cat=108&ccb=1-3&_nc_sid=1eb0c7&_nc_ohc=hekFCEl8ChsAX8rrxzZ&_nc_ht=scontent.frix7-1.fna&oh=c3c888d6f602307bd538cd3354c9dbae&oe=6128F92E"
},
"cover_url": {
"photo": {
"image": {
"uri": "https://scontent.frix7-1.fna.fbcdn.net/v/t1.6435-9/s960x960/170848750_5706897106016922_6014426456866089959_n.jpg?_nc_cat=108&ccb=1-3&_nc_sid=e3f864&_nc_ohc=P3Gc1aa6buIAX-SK6ox&_nc_oc=AQmAK6z3gO7_CXG6OVVrIIgkXT2lrFTmxewufu6tRLt9_58eO0BtTvNOS_EgEnEdo0g&_nc_ht=scontent.frix7-1.fna&oh=00ad207b73c7d828c589a6ad4fd8c6de&oe=612A8003"
},
"id": "5706897096016923"
}
},
"is_viewer_fan": false,
"can_viewer_like": false,
"should_show_page_like_warning": false
}
],
"can_viewer_see_transparency_card": true,
"transparency": {
"name_change_history_info": { "has_name_changed": true },
"date_change_history_info": { "has_date_changed": true }
}
}
]

7
src/#maldito-mapper.js#

@ -0,0 +1,7 @@
const input = require('./output.json');
const { by_date, event_times_to_dates, map_event } = require('./logic');
const events = input.map(map_event).map(event_times_to_dates).sort(by_date);
console.log(JSON.stringify(events));

17
src/facebook-event-payload-parser.js

@ -0,0 +1,17 @@
const pathOr = require('ramda/src/pathOr');
module.exports = {
parse_raw_string_to_event: (str) => {
if (str !== null) {
return str
.split('\n')
.filter((item) => item.length !== 0)
.map(JSON.parse)
.filter((item) => pathOr(null, ['path'], item) != null)
.filter((item) => item.path[0] == 'event')
.map(({ data }) => data)
.reduce((acc, item) => ({ ...acc, ...item }), {});
}
return null;
},
};

21
src/facebook-event.js

@ -1,13 +1,20 @@
const { get_event } = require('./logic'); const { get_event } = require('./logic');
const { do_request } = require('./facebook-request'); const { do_request } = require('./facebook-request');
const {
parse_raw_string_to_event,
} = require('./facebook-event-payload-parser');
const do_event_request = async (doc_id, event_id) => { const do_event_request = async (doc_id, event_id, parse = true) => {
let variables = { let variables = {
eventID: `${event_id}`, eventID: `${event_id}`,
}; };
const res = await do_request(doc_id, variables); const res = await do_request(doc_id, variables, parse);
return get_event(res); if (parse) {
return get_event(res);
} else {
return res;
}
}; };
/// EventsAboutTheVenueCardRendererQuery /// EventsAboutTheVenueCardRendererQuery
@ -28,11 +35,19 @@ const do_event_desc_request = (event_id) => {
return do_event_request(doc_id, event_id); return do_event_request(doc_id, event_id);
}; };
const do_event_time_request = async (event_id) => {
const doc_id = '3950443035053083';
const parse = false;
const str = await do_event_request(doc_id, event_id, parse);
return parse_raw_string_to_event(str);
};
const get_event_details = async (event_id) => { const get_event_details = async (event_id) => {
const res = await Promise.all([ const res = await Promise.all([
do_event_ticket_request(event_id), do_event_ticket_request(event_id),
do_event_location_request(event_id), do_event_location_request(event_id),
do_event_desc_request(event_id), do_event_desc_request(event_id),
do_event_time_request(event_id),
]); ]);
const event = res.reduce((acc, event) => ({ ...acc, ...event }), {}); const event = res.reduce((acc, event) => ({ ...acc, ...event }), {});
return event; return event;

65
src/facebook-page-events.js

@ -9,28 +9,12 @@ const {
const { do_request } = require('./facebook-request'); const { do_request } = require('./facebook-request');
const last = require('ramda/src/last'); const last = require('ramda/src/last');
const do_events_request = async (page_id, doc_id, cursor = null) => { const get_events = async (doc_id, variables, get_events_from_page, edges) => {
let variables = { let { cursor } = variables;
pageID: `${page_id}`,
allowedStates: ['PUBLISHED', 'CANCELED'],
cacheBreaker: 0,
scale: 1,
};
if (cursor !== null) {
variables = { ...variables, count: 20, cursor };
}
const res = await do_request(doc_id, variables);
return res;
};
const get_events = async (page_id, doc_id, get_events_from_page, events) => {
let { edges, cursor } = events;
let next = cursor !== null; let next = cursor !== null;
while (next) { while (next) {
let page; let page;
page = await do_events_request(page_id, doc_id, cursor); page = await do_request(doc_id, variables);
const { has_next_page } = get_page_info(page); const { has_next_page } = get_page_info(page);
page = get_events_from_page(page); page = get_events_from_page(page);
edges = [...edges, ...get_edges(page)]; edges = [...edges, ...get_edges(page)];
@ -41,13 +25,21 @@ const get_events = async (page_id, doc_id, get_events_from_page, events) => {
return edges; return edges;
}; };
const get_upcoming_events = async (page_id, events) => { const get_upcoming_events = async (page_id, { edges, cursor }) => {
const doc_id = '4766951026653856'; const doc_id = '4766951026653856';
const variables = {
count: 3,
cursor: cursor,
scale: 1,
id: `${page_id}`,
};
const upcoming_events = await get_events( const upcoming_events = await get_events(
page_id,
doc_id, doc_id,
variables,
get_upcoming_events_from_page, get_upcoming_events_from_page,
events, edges,
); );
return upcoming_events; return upcoming_events;
@ -55,20 +47,28 @@ const get_upcoming_events = async (page_id, events) => {
const get_past_events = async (page_id, events) => { const get_past_events = async (page_id, events) => {
const doc_id = '4082043558578171'; const doc_id = '4082043558578171';
const variables = {};
const past_events = await get_events( const past_events = await get_events(
page_id,
doc_id, doc_id,
variables,
get_past_events_from_page, get_past_events_from_page,
events, events,
); );
return past_events; return past_events;
}; };
const get_initial_events = (page_id) => {
const doc_id = '4071780429584964';
const variables = { pageID: `${page_id}`, scale: 1 };
return do_request(doc_id, variables);
};
const get_reoccuring_events = () => {}; const get_reoccuring_events = () => {};
const init_scrape = async (page_id) => { const init_scrape = async (page_id) => {
const doc_id = '4071780429584964'; const res = await get_initial_events(page_id);
const res = await do_events_request(page_id, doc_id);
const upcoming_events = get_upcoming_events_from_page(res); const upcoming_events = get_upcoming_events_from_page(res);
const past_events = get_past_events_from_page(res); const past_events = get_past_events_from_page(res);
@ -96,12 +96,15 @@ const get_page_events = async (opt) => {
let { upcoming_events, past_events } = await init_scrape(page_id); let { upcoming_events, past_events } = await init_scrape(page_id);
if (opt.get_past_events) { /* if (opt.get_past_events) {
await sleep(2); * await sleep(2);
past_events = await get_past_events(page_id, past_events); * past_events = await get_past_events(page_id, past_events);
} else { * } else {
past_events = []; * past_events = [];
} * }
*/
past_events = [];
if (opt.get_upcoming_events) { if (opt.get_upcoming_events) {
await sleep(2); await sleep(2);

54
src/facebook-payload.txt

File diff suppressed because one or more lines are too long

23
src/facebook-request.js

@ -3,7 +3,7 @@ const graphql_endpoint = 'https://www.facebook.com/api/graphql/';
const https_proxy_agent = require('https-proxy-agent'); const https_proxy_agent = require('https-proxy-agent');
const do_request = async (doc_id, variables) => { const do_request = async (doc_id, variables, parse = true) => {
const params = new URLSearchParams(); const params = new URLSearchParams();
const agent = new https_proxy_agent('http://10.0.0.210:5566'); const agent = new https_proxy_agent('http://10.0.0.210:5566');
params.append('doc_id', doc_id); params.append('doc_id', doc_id);
@ -30,13 +30,22 @@ const do_request = async (doc_id, variables) => {
return null; return null;
} }
try { const txt = await res.text();
res = await res.json();
} catch (e) { if (parse) {
console.error(e); try {
return null; res = JSON.parse(txt);
} catch (e) {
console.error(e);
return null;
}
} else {
if (txt.length == 0) {
return null;
} else {
return txt;
}
} }
return res; return res;
}; };

2
src/logic.js

@ -119,6 +119,8 @@ const map_event = (event) => {
const end = pathOr(0, ['end_timestamp'], event) * 1000; const end = pathOr(0, ['end_timestamp'], event) * 1000;
const updated = pathOr(0, ['updated_time'], event) * 1000; const updated = pathOr(0, ['updated_time'], event) * 1000;
console.log(event);
return { return {
canceled, canceled,
host, host,

6
src/maldito-mapper.js

@ -2,10 +2,6 @@ const input = require('./output.json');
const { by_date, event_times_to_dates, map_event } = require('./logic'); const { by_date, event_times_to_dates, map_event } = require('./logic');
const events = input const events = input.map(map_event).map(event_times_to_dates).sort(by_date);
.map(map_event)
.map(event_times_to_dates)
.sort(by_date)
.map(({ date, ...event }) => ({ ...event, date }));
console.log(JSON.stringify(events)); console.log(JSON.stringify(events));

2636
src/output.json

File diff suppressed because it is too large Load Diff

8
src/scrape-event.js

@ -23,16 +23,18 @@ const options = parse_args(process.argv.slice(2));
if (index === -1) { if (index === -1) {
sleep(2); sleep(2);
console.error('INFO: New event. Fetching details.'); console.error(
`INFO: New event. ${event.name}, fetching details. ${event.url}`,
);
const event_details = await get_event_details(event.id); const event_details = await get_event_details(event.id);
events.push({ ...event, ...event_details }); events.push({ ...event, ...event_details });
} else if (previous_events[index].updated_time != event.updated_time) { } else if (previous_events[index].updated_time != event.updated_time) {
sleep(2); sleep(2);
console.error('INFO: Event needs updating.'); console.error(`INFO: Event needs ${event.name} updating. ${event.url}`);
const event_details = await get_event_details(event.id); const event_details = await get_event_details(event.id);
events.push({ ...event, ...event_details }); events.push({ ...event, ...event_details });
} else { } else {
console.error('INFO: Using already scraped event.'); console.error(`INFO: Event ${event.name} already scraped. ${event.url}`);
events.push(previous_events[index]); events.push(previous_events[index]);
} }
} }

4574
yarn.lock

File diff suppressed because it is too large Load Diff
Loading…
Cancel
Save