Browse Source

add working version of scraper with maldito powers

fix-broken-scrape
Jørgen Sverre Lien Sellæg 5 years ago
parent
commit
c6bd07d44c
  1. 2636
      backup.json
  2. 28
      src/logic.js
  3. 7
      src/maldito-mapper.js

2636
backup.json

File diff suppressed because it is too large Load Diff

28
src/logic.js

@ -89,11 +89,8 @@ const to_unique_events = (acc, current) => [
current, current,
]; ];
const get_city_name = (event) => const get_city_name = (event) => pathOr('', ['place', 'city', 'name'], event);
pathOr('', ['event_place', 'city', 'contextual_name'], event); const get_host_name = (event) => pathOr('', ['place', 'name'], event);
const get_event_host = (event) =>
pathOr('', ['event_place', 'contextual_name'], event);
const read_previous_events = (path) => { const read_previous_events = (path) => {
if (path !== null) { if (path !== null) {
@ -113,12 +110,13 @@ const read_previous_events = (path) => {
const write_events = (path, events) => const write_events = (path, events) =>
fs.writeFile(path, JSON.stringify(events)); fs.writeFile(path, JSON.stringify(events));
const map_event = ({ node: event }) => { const map_event = (event) => {
const ticket_url = pathOr('', ['event_buy_ticket_url'], event); const ticket_url = pathOr('', ['ticketUrl'], event);
const city = get_city_name(event); const city = get_city_name(event);
const host = get_event_host(event); const host = get_host_name(event);
const canceled = pathOr(false, ['is_canceled'], event); const canceled = pathOr(false, ['is_canceled'], event);
const start = pathOr(0, ['start_timestamp'], event) * 1000; const start = pathOr(0, ['start_timestamp'], event) * 1000;
const end = pathOr(0, ['end_timestamp'], event) * 1000;
const updated = pathOr(0, ['updated_time'], event) * 1000; const updated = pathOr(0, ['updated_time'], event) * 1000;
return { return {
@ -128,7 +126,7 @@ const map_event = ({ node: event }) => {
location: city, location: city,
name: event.name, name: event.name,
ticket_url, ticket_url,
date: { start, updated }, date: { start, updated, end },
}; };
}; };
@ -151,6 +149,17 @@ const by_date = (a, b) => {
return -1; return -1;
}; };
const event_times_to_dates = ({ date, ...event }) => {
return {
date: {
start: new Date(date.start),
end: new Date(date.end),
updated: new Date(date.updated),
},
...event,
};
};
const event_date_to_date_obj = (event) => { const event_date_to_date_obj = (event) => {
const start = pathOr(null, ['date', 'start'], event); const start = pathOr(null, ['date', 'start'], event);
const updated = pathOr(null, ['date', 'updated'], event); const updated = pathOr(null, ['date', 'updated'], event);
@ -169,6 +178,7 @@ const event_date_to_date_obj = (event) => {
const sleep = (s) => new Promise((res) => setTimeout(res, s * 1000)); const sleep = (s) => new Promise((res) => setTimeout(res, s * 1000));
module.exports = { module.exports = {
event_times_to_dates,
by_date, by_date,
event_date_to_date_obj, event_date_to_date_obj,
get_edges, get_edges,

7
src/maldito-mapper.js

@ -0,0 +1,7 @@
const input = require('../output.json');
const { by_date, event_times_to_dates, map_event } = require('./logic');
const events = input.map(map_event).map(event_times_to_dates).sort(by_date);
console.log(JSON.stringify(events));
Loading…
Cancel
Save