add hoopla scraper

bin/hoopla.mjs

@@ -0,0 +1,196 @@
+import { get_upcoming_events } from '../src/hoopla/index.mjs';
+import send from '../src/signal/send.mjs';
+import fetch from 'node-fetch';
+
+const prod = false;
+
+let api, token;
+
+if (prod) {
+  api = 'http://10.0.0.210:8484';
+  token = '831411806230c7e950c4eeb226499ef92bb6bdc4157797929a0e16d133dc13a8';
+} else {
+  api = 'http://localhost:3333';
+  token = '1234567812345678123456781234567812345678123456781234567812345678';
+}
+
+const headers = { 'Content-Type': 'application/json' };
+
+const scrape = async (pageID) => {
+  try {
+    const res = await get_page_events({
+      pageID,
+      get_upcoming_events: true,
+      get_past_events: false
+    });
+    return res;
+  } catch (e) {
+    console.error(e);
+  }
+  return [];
+};
+const unix = (a) => parseInt(new Date(a).valueOf() / 1000, 10);
+const sleep = (ms) => new Promise((res) => setTimeout(res, ms));
+const updated = (oldEvent, scrapedEvent) => {
+  let keys = [
+    'canceled',
+    'end',
+    'start',
+    'draft',
+    'facebook_id',
+    'place_id',
+    'name',
+    'ticket_url'
+  ];
+  for (let key of keys) {
+    if (oldEvent[key] != scrapedEvent[key]) {
+      console.log(124, oldEvent[key], '!=', scrapedEvent[key]);
+      return true;
+    }
+  }
+  return false;
+};
+
+(async () => {
+  let resp = await fetch(`${api}/places/?token=${token}`);
+  let places = await resp.json();
+  places = places.filter((place) => {
+    const scrape = place.scraper == 'hoopla';
+    if (!scrape) {
+      console.log(
+        101,
+        `Skipping #${place.id} ${place.name}. Reason: Scraper is ${place.scraper}`
+      );
+      return false;
+    }
+    const now = unix(new Date());
+    const recently = place.last_scraped + place.scrape_threshold;
+    if (now < recently) {
+      console.log(
+        100,
+        `Skipping #${place.id} ${place.name}. Reason: Was scraped ${
+          now - place.last_scraped
+        }s ago.`
+      );
+      return false;
+    }
+    return true;
+  });
+
+  for (let place of places) {
+    console.log(177, `Scraping #${place.id} ${place.name}`);
+    const events = await get_upcoming_events(place.hoopla_id);
+    let payloads = [];
+    for (let event of events) {
+      payloads.push({
+        canceled: false,
+        end: unix(new Date(event.end)),
+        start: unix(new Date(event.start)),
+        draft: false,
+        hoopla_id: event.event_id,
+        place_id: place.id,
+        name: event.name ?? '',
+        ticket_url: `https://${hoopla_name_id}.hoopla.no/sales/${event.event_id}`
+      });
+    }
+    if (payloads.length == 0) {
+      console.log(123, 'No upcoming events, dead place?');
+    }
+    for (let payload of payloads) {
+      let search = await fetch(
+        `${api}/search/events/?hoopla_id=${payload.hoopla_id}&token=${token}`
+      );
+      if (!search.ok) {
+        console.log(500, await search.text());
+        continue;
+      }
+      search = await search.json();
+      let new_event = search.length === 0;
+      let old_event;
+      if (!new_event) {
+        old_event = search[0];
+      }
+      let res;
+      if (new_event) {
+        res = await fetch(`${api}/events/?token=${token}`, {
+          method: 'POST',
+          body: JSON.stringify(payload),
+          headers
+        });
+        console.log(res.status, 'Insert', place.name, payload.name);
+        let newEvent = await res.text();
+        let msg = await send(newEvent, place);
+        console.log(res.status, 'Signal', msg);
+      } else if (old_event && updated(old_event, payload)) {
+        payload.id = old_event.id;
+        if (old_event.ticket_url.length > 0 && payload.ticket_url.length == 0) {
+          payload.ticket_url = old_event.ticket_url;
+        }
+        if (updated(old_event, payload)) {
+          res = await fetch(`${api}/events/${old_event.id}/?token=${token}`, {
+            method: 'PATCH',
+            body: JSON.stringify(payload),
+            headers
+          });
+          console.log(res.status, 'Update', place.name, payload.name);
+        } else {
+          console.log(201, 'Skip Update', place.name, payload.name);
+        }
+      } else {
+        console.log(201, 'Skip', place.name, payload.name);
+      }
+    }
+    let res = await fetch(`${api}/places/${place.id}/?token=${token}`, {
+      method: 'PATCH',
+      body: JSON.stringify({
+        last_scraped: unix(new Date())
+      }),
+      headers
+    });
+    console.log(res.status, `Last scrape at ${place.name} updated.`);
+  }
+})();
+
+let example = {
+  event_id: 143146107,
+  organization_id: 1947342940,
+  identifier: 'vvalentinerne',
+  name: 'Vidar & Valentinerne',
+  description:
+    'Vidar & Valentinerne\nLobbyen - 18. års aldersgrense. \n\nVALENTINERNE (Oslo) spilte sammen med Joachim «Jokke» Nielsen på 80- og 90-tallet, og består av May-Irene Aasen (trommer), Petter Pogo (gitar), Håkon Torgersen (bass) og de har med seg selveste Vidar Rugset på vokal og gitar. Sammen fremfører de Jokke & Valentinernes musikk på nær autentisk vis. ',
+  start: '2022-08-12T19:00:00Z',
+  end: '2022-08-12T23:00:00Z',
+  data: {
+    location: {
+      name: 'Verkstedhallen & Lobbyen',
+      street_address: 'Strandveien 29',
+      postal_code: '7067',
+      postal_area: 'Trondheim'
+    },
+    image: '1947342940/vidar-valentinerne.1654597144.jpg',
+    image_crop: {
+      percentTop: 1.97,
+      percentBottom: 1.59,
+      percentLeft: 0,
+      percentRight: 0,
+      width: 628,
+      height: 392.5,
+      x: 0,
+      y: 8.02
+    },
+    max_tickets: 10,
+    category: 'CONCERT',
+    other_category_description: ''
+  },
+  is_published: true,
+  published_at: '2022-06-07T10:37:40.817271Z',
+  is_cancelled: false,
+  invoice_allowed: false,
+  has_slatejs_description: false,
+  has_promo_codes: false,
+  has_addons: true,
+  created: '2022-06-07T10:19:50.6722Z',
+  featured: null,
+  feature_priority: null,
+  images: null
+};

bin/hoopla.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+readonly SCRIPT_HOME=$(dirname `readlink -f $0`)
+export NODE_EXTRA_CA_CERTS="$SCRIPT_HOME/../share/ca.crt"
+
+function run {
+  cd $SCRIPT_HOME
+  node ./hoopla.mjs
+}
+
+run

src/hoopla/index.mjs

@@ -0,0 +1,21 @@
+import fetch from 'node-fetch';
+
+const api = 'https://hoopla.no/api/v2.0/public/organizations/';
+
+export const get_upcoming_events = async (pageID) => {
+  let res = null;
+  try {
+    res = await fetch(`${api}/${pageID}/events`);
+    if (!res.ok) {
+      return [];
+    }
+    res = await res.json();
+    return res?.data ?? [];
+  } catch (e) {
+    console.error(e);
+    return [];
+  }
+  return res;
+};
+
+export default get_upcoming_events;