From a598bb1cdd76eb101c9ef045ba51bb4ccc9fc063 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rgen=20Sverre=20Lien=20Sell=C3=A6g?= Date: Fri, 8 Jul 2022 22:58:20 +0200 Subject: [PATCH] add hoopla scraper --- bin/hoopla.mjs | 196 +++++++++++++++++++++++++++++++++++++++++++ bin/hoopla.sh | 10 +++ src/hoopla/index.mjs | 21 +++++ 3 files changed, 227 insertions(+) create mode 100644 bin/hoopla.mjs create mode 100755 bin/hoopla.sh create mode 100644 src/hoopla/index.mjs diff --git a/bin/hoopla.mjs b/bin/hoopla.mjs new file mode 100644 index 0000000..11b9c54 --- /dev/null +++ b/bin/hoopla.mjs @@ -0,0 +1,196 @@ +import { get_upcoming_events } from '../src/hoopla/index.mjs'; +import send from '../src/signal/send.mjs'; +import fetch from 'node-fetch'; + +const prod = false; + +let api, token; + +if (prod) { + api = 'http://10.0.0.210:8484'; + token = '831411806230c7e950c4eeb226499ef92bb6bdc4157797929a0e16d133dc13a8'; +} else { + api = 'http://localhost:3333'; + token = '1234567812345678123456781234567812345678123456781234567812345678'; +} + +const headers = { 'Content-Type': 'application/json' }; + +const scrape = async (pageID) => { + try { + const res = await get_page_events({ + pageID, + get_upcoming_events: true, + get_past_events: false + }); + return res; + } catch (e) { + console.error(e); + } + return []; +}; +const unix = (a) => parseInt(new Date(a).valueOf() / 1000, 10); +const sleep = (ms) => new Promise((res) => setTimeout(res, ms)); +const updated = (oldEvent, scrapedEvent) => { + let keys = [ + 'canceled', + 'end', + 'start', + 'draft', + 'facebook_id', + 'place_id', + 'name', + 'ticket_url' + ]; + for (let key of keys) { + if (oldEvent[key] != scrapedEvent[key]) { + console.log(124, oldEvent[key], '!=', scrapedEvent[key]); + return true; + } + } + return false; +}; + +(async () => { + let resp = await fetch(`${api}/places/?token=${token}`); + let places = await resp.json(); + places = places.filter((place) => { + const scrape = place.scraper == 'hoopla'; + if (!scrape) { + console.log( + 101, + `Skipping #${place.id} ${place.name}. Reason: Scraper is ${place.scraper}` + ); + return false; + } + const now = unix(new Date()); + const recently = place.last_scraped + place.scrape_threshold; + if (now < recently) { + console.log( + 100, + `Skipping #${place.id} ${place.name}. Reason: Was scraped ${ + now - place.last_scraped + }s ago.` + ); + return false; + } + return true; + }); + + for (let place of places) { + console.log(177, `Scraping #${place.id} ${place.name}`); + const events = await get_upcoming_events(place.hoopla_id); + let payloads = []; + for (let event of events) { + payloads.push({ + canceled: false, + end: unix(new Date(event.end)), + start: unix(new Date(event.start)), + draft: false, + hoopla_id: event.event_id, + place_id: place.id, + name: event.name ?? '', + ticket_url: `https://${hoopla_name_id}.hoopla.no/sales/${event.event_id}` + }); + } + if (payloads.length == 0) { + console.log(123, 'No upcoming events, dead place?'); + } + for (let payload of payloads) { + let search = await fetch( + `${api}/search/events/?hoopla_id=${payload.hoopla_id}&token=${token}` + ); + if (!search.ok) { + console.log(500, await search.text()); + continue; + } + search = await search.json(); + let new_event = search.length === 0; + let old_event; + if (!new_event) { + old_event = search[0]; + } + let res; + if (new_event) { + res = await fetch(`${api}/events/?token=${token}`, { + method: 'POST', + body: JSON.stringify(payload), + headers + }); + console.log(res.status, 'Insert', place.name, payload.name); + let newEvent = await res.text(); + let msg = await send(newEvent, place); + console.log(res.status, 'Signal', msg); + } else if (old_event && updated(old_event, payload)) { + payload.id = old_event.id; + if (old_event.ticket_url.length > 0 && payload.ticket_url.length == 0) { + payload.ticket_url = old_event.ticket_url; + } + if (updated(old_event, payload)) { + res = await fetch(`${api}/events/${old_event.id}/?token=${token}`, { + method: 'PATCH', + body: JSON.stringify(payload), + headers + }); + console.log(res.status, 'Update', place.name, payload.name); + } else { + console.log(201, 'Skip Update', place.name, payload.name); + } + } else { + console.log(201, 'Skip', place.name, payload.name); + } + } + let res = await fetch(`${api}/places/${place.id}/?token=${token}`, { + method: 'PATCH', + body: JSON.stringify({ + last_scraped: unix(new Date()) + }), + headers + }); + console.log(res.status, `Last scrape at ${place.name} updated.`); + } +})(); + +let example = { + event_id: 143146107, + organization_id: 1947342940, + identifier: 'vvalentinerne', + name: 'Vidar & Valentinerne', + description: + 'Vidar & Valentinerne\nLobbyen - 18. års aldersgrense. \n\nVALENTINERNE (Oslo) spilte sammen med Joachim «Jokke» Nielsen på 80- og 90-tallet, og består av May-Irene Aasen (trommer), Petter Pogo (gitar), Håkon Torgersen (bass) og de har med seg selveste Vidar Rugset på vokal og gitar. Sammen fremfører de Jokke & Valentinernes musikk på nær autentisk vis. ', + start: '2022-08-12T19:00:00Z', + end: '2022-08-12T23:00:00Z', + data: { + location: { + name: 'Verkstedhallen & Lobbyen', + street_address: 'Strandveien 29', + postal_code: '7067', + postal_area: 'Trondheim' + }, + image: '1947342940/vidar-valentinerne.1654597144.jpg', + image_crop: { + percentTop: 1.97, + percentBottom: 1.59, + percentLeft: 0, + percentRight: 0, + width: 628, + height: 392.5, + x: 0, + y: 8.02 + }, + max_tickets: 10, + category: 'CONCERT', + other_category_description: '' + }, + is_published: true, + published_at: '2022-06-07T10:37:40.817271Z', + is_cancelled: false, + invoice_allowed: false, + has_slatejs_description: false, + has_promo_codes: false, + has_addons: true, + created: '2022-06-07T10:19:50.6722Z', + featured: null, + feature_priority: null, + images: null +}; diff --git a/bin/hoopla.sh b/bin/hoopla.sh new file mode 100755 index 0000000..34cd09f --- /dev/null +++ b/bin/hoopla.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +readonly SCRIPT_HOME=$(dirname `readlink -f $0`) +export NODE_EXTRA_CA_CERTS="$SCRIPT_HOME/../share/ca.crt" + +function run { + cd $SCRIPT_HOME + node ./hoopla.mjs +} + +run diff --git a/src/hoopla/index.mjs b/src/hoopla/index.mjs new file mode 100644 index 0000000..43a2c3d --- /dev/null +++ b/src/hoopla/index.mjs @@ -0,0 +1,21 @@ +import fetch from 'node-fetch'; + +const api = 'https://hoopla.no/api/v2.0/public/organizations/'; + +export const get_upcoming_events = async (pageID) => { + let res = null; + try { + res = await fetch(`${api}/${pageID}/events`); + if (!res.ok) { + return []; + } + res = await res.json(); + return res?.data ?? []; + } catch (e) { + console.error(e); + return []; + } + return res; +}; + +export default get_upcoming_events;