Compare commits
No commits in common. 'master' and 'fix-broken-scrape' have entirely different histories.
master
...
fix-broken
10 changed files with 68 additions and 291 deletions
@ -1,174 +0,0 @@
|
||||
import { get_upcoming_events } from '../src/hoopla/index.mjs'; |
||||
import send from '../src/signal/send.mjs'; |
||||
import { get_credentials, updated } from '../src/util.mjs'; |
||||
import fetch from 'node-fetch'; |
||||
|
||||
const prod = true; |
||||
|
||||
let { api, token } = get_credentials(prod); |
||||
|
||||
if (prod) { |
||||
api = 'http://10.0.0.210:8484'; |
||||
token = '831411806230c7e950c4eeb226499ef92bb6bdc4157797929a0e16d133dc13a8'; |
||||
} else { |
||||
api = 'http://localhost:3333'; |
||||
token = '1234567812345678123456781234567812345678123456781234567812345678'; |
||||
} |
||||
|
||||
const headers = { 'Content-Type': 'application/json' }; |
||||
const unix = (a) => parseInt(new Date(a).valueOf() / 1000, 10); |
||||
|
||||
(async () => { |
||||
let resp = await fetch(`${api}/places/?token=${token}`); |
||||
let places = await resp.json(); |
||||
places = places.filter((place) => { |
||||
const scrape = place.scraper == 'hoopla'; |
||||
if (!scrape) { |
||||
// console.log(
|
||||
// 101,
|
||||
// `Skipping #${place.id} ${place.name}. Reason: Scraper is ${place.scraper}`
|
||||
// );
|
||||
return false; |
||||
} |
||||
const now = unix(new Date()); |
||||
const recently = place.last_scraped + place.scrape_threshold; |
||||
if (now < recently) { |
||||
console.log( |
||||
100, |
||||
`Skipping #${place.id} ${place.name}. Reason: Was scraped ${ |
||||
now - place.last_scraped |
||||
}s ago.` |
||||
); |
||||
return false; |
||||
} |
||||
return true; |
||||
}); |
||||
|
||||
for (let place of places) { |
||||
console.log(177, `Scraping #${place.id} ${place.name}`); |
||||
const events = await get_upcoming_events(place.hoopla_id); |
||||
let payloads = []; |
||||
for (let event of events) { |
||||
payloads.push({ |
||||
canceled: false, |
||||
end: unix(new Date(event.end)), |
||||
start: unix(new Date(event.start)), |
||||
draft: false, |
||||
hoopla_id: `${event.event_id}`, |
||||
facebook_id: '', |
||||
place_id: place.id, |
||||
name: event.name ?? '', |
||||
ticket_url: `https://${place.hoopla_name_id}.hoopla.no/sales/${event.event_id}` |
||||
}); |
||||
} |
||||
if (payloads.length == 0) { |
||||
console.log(123, 'No upcoming events, dead place?'); |
||||
} |
||||
for (let payload of payloads) { |
||||
let search; |
||||
search = `${api}/search/events/?hoopla_id=${payload.hoopla_id}&token=${token}`; |
||||
console.log(search); |
||||
search = await fetch(search); |
||||
if (!search.ok) { |
||||
console.log(search.status, 'Search was not OK.', await search.text()); |
||||
continue; |
||||
} |
||||
search = await search.json(); |
||||
let new_event = search.length === 0; |
||||
let old_event; |
||||
if (!new_event) { |
||||
old_event = search[0]; |
||||
} |
||||
let res; |
||||
if (new_event) { |
||||
res = await fetch(`${api}/events/?token=${token}`, { |
||||
method: 'POST', |
||||
body: JSON.stringify(payload), |
||||
headers |
||||
}); |
||||
console.log(res.status, 'Insert', place.name, payload.name); |
||||
let newEvent = await res.text(); |
||||
let msg = await send(newEvent, place); |
||||
console.log(res.status, 'Signal', msg); |
||||
} else if (old_event && updated(old_event, payload)) { |
||||
payload.id = old_event.id; |
||||
if (old_event.ticket_url.length > 0 && payload.ticket_url.length == 0) { |
||||
payload.ticket_url = old_event.ticket_url; |
||||
} |
||||
if (updated(old_event, payload)) { |
||||
res = await fetch(`${api}/events/${old_event.id}/?token=${token}`, { |
||||
method: 'PATCH', |
||||
body: JSON.stringify(payload), |
||||
headers |
||||
}); |
||||
console.log(res.status, 'Update', place.name, payload.name); |
||||
} else { |
||||
console.log(201, 'Skip Update', place.name, payload.name); |
||||
} |
||||
} else { |
||||
console.log(201, 'Skip', place.name, payload.name); |
||||
} |
||||
} |
||||
if (payloads.length > 0) { |
||||
let res = await fetch(`${api}/places/${place.id}/?token=${token}`, { |
||||
method: 'PATCH', |
||||
body: JSON.stringify({ |
||||
last_scraped: unix(new Date()) |
||||
}), |
||||
headers |
||||
}); |
||||
if (res.ok) { |
||||
console.log(res.status, `Last scrape at ${place.name} updated.`); |
||||
} else { |
||||
console.log( |
||||
res.status, |
||||
`Last scrape at ${place.name} failed to update last update..` |
||||
); |
||||
} |
||||
} |
||||
} |
||||
})(); |
||||
|
||||
let example = { |
||||
event_id: 143146107, |
||||
organization_id: 1947342940, |
||||
identifier: 'vvalentinerne', |
||||
name: 'Vidar & Valentinerne', |
||||
description: |
||||
'Vidar & Valentinerne\nLobbyen - 18. års aldersgrense. \n\nVALENTINERNE (Oslo) spilte sammen med Joachim «Jokke» Nielsen på 80- og 90-tallet, og består av May-Irene Aasen (trommer), Petter Pogo (gitar), Håkon Torgersen (bass) og de har med seg selveste Vidar Rugset på vokal og gitar. Sammen fremfører de Jokke & Valentinernes musikk på nær autentisk vis. ', |
||||
start: '2022-08-12T19:00:00Z', |
||||
end: '2022-08-12T23:00:00Z', |
||||
data: { |
||||
location: { |
||||
name: 'Verkstedhallen & Lobbyen', |
||||
street_address: 'Strandveien 29', |
||||
postal_code: '7067', |
||||
postal_area: 'Trondheim' |
||||
}, |
||||
image: '1947342940/vidar-valentinerne.1654597144.jpg', |
||||
image_crop: { |
||||
percentTop: 1.97, |
||||
percentBottom: 1.59, |
||||
percentLeft: 0, |
||||
percentRight: 0, |
||||
width: 628, |
||||
height: 392.5, |
||||
x: 0, |
||||
y: 8.02 |
||||
}, |
||||
max_tickets: 10, |
||||
category: 'CONCERT', |
||||
other_category_description: '' |
||||
}, |
||||
is_published: true, |
||||
published_at: '2022-06-07T10:37:40.817271Z', |
||||
is_cancelled: false, |
||||
invoice_allowed: false, |
||||
has_slatejs_description: false, |
||||
has_promo_codes: false, |
||||
has_addons: true, |
||||
created: '2022-06-07T10:19:50.6722Z', |
||||
featured: null, |
||||
feature_priority: null, |
||||
images: null |
||||
}; |
||||
@ -1,11 +0,0 @@
|
||||
#!/usr/bin/env bash |
||||
readonly SCRIPT_HOME=$(dirname `readlink -f $0`) |
||||
export NODE_EXTRA_CA_CERTS="$SCRIPT_HOME/../share/ca.crt" |
||||
export NODE_TLS_REJECT_UNAUTHORIZED='0' |
||||
|
||||
function run { |
||||
cd $SCRIPT_HOME |
||||
node ./hoopla.mjs |
||||
} |
||||
|
||||
run |
||||
@ -1,21 +0,0 @@
|
||||
import fetch from 'node-fetch'; |
||||
import { unix, get_credentials } from '../util.mjs'; |
||||
|
||||
export const update_last_scraped = async (place, prod = false) => { |
||||
const { api, token } = get_credentials(prod); |
||||
let res = await fetch(`${api}/places/${place.id}/?token=${token}`, { |
||||
method: 'PATCH', |
||||
body: JSON.stringify({ |
||||
last_scraped: unix(new Date()) |
||||
}), |
||||
headers: { 'Content-Type': 'application/json' } |
||||
}); |
||||
if (res.ok) { |
||||
console.log(res.status, `Last scrape at ${place.name} updated.`); |
||||
} else { |
||||
console.log( |
||||
res.status, |
||||
`Last scrape at ${place.name} failed to update last update..` |
||||
); |
||||
} |
||||
}; |
||||
@ -1,21 +0,0 @@
|
||||
import fetch from 'node-fetch'; |
||||
|
||||
const api = 'https://hoopla.no/api/v2.0/public/organizations'; |
||||
|
||||
export const get_upcoming_events = async (pageID) => { |
||||
let res = null; |
||||
try { |
||||
res = await fetch(`${api}/${pageID}/events`); |
||||
if (!res.ok) { |
||||
return []; |
||||
} |
||||
res = await res.json(); |
||||
return res?.data ?? []; |
||||
} catch (e) { |
||||
console.error(e); |
||||
return []; |
||||
} |
||||
return res; |
||||
}; |
||||
|
||||
export default get_upcoming_events; |
||||
@ -1,36 +0,0 @@
|
||||
export const get_credentials = (prod = false) => { |
||||
let api, token; |
||||
if (prod) { |
||||
api = 'http://10.0.0.210:8484'; |
||||
token = '831411806230c7e950c4eeb226499ef92bb6bdc4157797929a0e16d133dc13a8'; |
||||
} else { |
||||
api = 'http://localhost:3333'; |
||||
token = '1234567812345678123456781234567812345678123456781234567812345678'; |
||||
} |
||||
return { api, token }; |
||||
}; |
||||
export const sleep = (ms) => { |
||||
return new Promise((res) => setTimeout(res, ms)); |
||||
}; |
||||
export const unix = (a) => { |
||||
return parseInt(new Date(a).valueOf() / 1000, 10); |
||||
}; |
||||
export const updated = (oldEvent, scrapedEvent) => { |
||||
let keys = [ |
||||
'canceled', |
||||
'end', |
||||
'start', |
||||
'draft', |
||||
'facebook_id', |
||||
'place_id', |
||||
'name', |
||||
'ticket_url' |
||||
]; |
||||
for (let key of keys) { |
||||
if (oldEvent[key] != scrapedEvent[key]) { |
||||
console.log(124, key, oldEvent[key], '!=', scrapedEvent[key]); |
||||
return true; |
||||
} |
||||
} |
||||
return false; |
||||
}; |
||||
Loading…
Reference in new issue