Browse Source

better handling of fetching places

fix-broken-scrape
Jørgen Sverre Lien Sellæg 4 years ago
parent
commit
d9d095f93a
  1. 143
      bin/adapter.mjs

143
bin/adapter.mjs

@ -6,63 +6,150 @@ const later = (delay, value) =>
new Promise((resolve) => setTimeout(resolve, delay, value)); new Promise((resolve) => setTimeout(resolve, delay, value));
const unix = (a) => parseInt(new Date(a).valueOf() / 1000, 10); const unix = (a) => parseInt(new Date(a).valueOf() / 1000, 10);
const token =
'831411806230c7e950c4eeb226499ef92bb6bdc4157797929a0e16d133dc13a8';
const hostname = 'https://dansbart.no';
// const hostname = 'http://localhost:3333';
const get_city = (event) => {
let trondheim =
event.event_place?.city?.id === '110429825645017' ||
event.place?.city?.id === '110429825645017';
if (trondheim) {
return 'Trondheim, Norway';
}
let city = event.event_place?.city?.name ?? null;
if (city === null) {
city = event.event_place?.city?.contextual_name ?? null;
}
if (city === null) {
city = event.place?.city?.name ?? null;
}
if (city === null) {
city = event.place?.city?.contextual_name ?? null;
}
if (city !== null) {
return city;
}
return 'Trondheim, Norway';
};
const get_place = (event) => {
let place, address, page;
address =
event.event_place?.__typename === 'FreeformPlace' ||
event.place?.__typename === 'FreeformPlace';
page =
event.event_place?.__typename === 'Page' ||
event.place?.__typename === 'Page';
if (address) {
place = event?.event_place?.contextual_name ?? null;
if (place === null) {
place = event?.place?.contextual_name ?? null;
}
if (place == null) {
place = '';
}
} else if (page) {
place = event?.event_place?.name ?? null;
if (place === null) {
place = event?.place?.name ?? null;
}
if (place === null) {
place = event?.event_place?.contextual_name ?? null;
}
if (place === null) {
place = event?.place?.contextual_name ?? null;
}
if (place == null) {
place == '';
}
} else {
place = '';
}
if (place === '' || place == null) {
console.log(event);
console.log(place);
if (place === undefined) {
process.exit(1);
}
}
return place;
};
const sleep = (ms) => new Promise((res) => setTimeout(res, ms));
(async () => { (async () => {
const root = '/home/zalox/src/acne/lib/derma'; const root = '/home/zalox/src/acne/lib/derma';
let requests = []; let requests = [];
const promises = eventFiles.map(async (filename) => { const promises = eventFiles.map(async (filename) => {
const path = `${root}/${filename}`; const path = `${root}/${filename}`;
const file_content = await fs.readFile(path, { encoding: 'utf-8' }); const file_content = await fs.readFile(path, { encoding: 'utf-8' });
const j = JSON.parse(file_content); const j = JSON.parse(file_content);
for (let i = 0; i < j.length; i++) { for (let i = 0; i < j.length; i++) {
let event = j[i]; let event = j[i];
let trondheim = event.event_place?.city?.id === '110429825645017';
const payload = { const payload = {
canceled: false, canceled: false,
end: -1, end: -1,
start: unix(new Date(event.time_range.start)), start: unix(new Date(event.time_range.start)),
draft: false, draft: false,
facebook_id: event.id, facebook_id: event.id,
host: event.event_place?.name ?? '', host: get_place(event),
location: trondheim location: get_city(event),
? 'Trondheim, Norway'
: event.event_place?.city?.contextual_name ?? 'Trondheim, Norway',
name: event.name ?? 'NAME_MISSING', name: event.name ?? 'NAME_MISSING',
ticket_url: event.event_buy_ticket_url ?? '' ticket_url: event.event_buy_ticket_url ?? ''
}; };
requests.push(payload); requests.push(payload);
} }
}); });
await Promise.all(promises); await Promise.all(promises);
for (let i = 0; i < requests.length; i++) { for (let i = 0; i < requests.length; i++) {
let event = requests[i]; let event = requests[i];
const res = await fetch( let search = await fetch(
'https://dansbart.no/events/?token=831411806230c7e950c4eeb226499ef92bb6bdc4157797929a0e16d133dc13a8', `${hostname}/search/events/?facebook_id=${event.facebook_id}&token=${token}`
{ );
if (!search.ok) {
console.error(117, search.status);
console.error(118, 'search not ok');
console.error(119, await search.text());
await sleep(1000);
continue;
}
search = await search.json();
let update = search.length === 1;
let res;
if (update) {
let old_event = search[0];
event.id = old_event.id;
if (old_event.ticket_url.length > 0 && event.ticket_url.length == 0) {
delete event.ticket_url;
}
console.error('132', event);
res = await fetch(`${hostname}/events/${event.id}/?token=${token}`, {
method: 'PATCH',
body: JSON.stringify(event),
headers: { 'Content-Type': 'application/json' }
});
console.error(138, await res.text());
console.error(139, 'update');
} else {
res = await fetch(`${hostname}/events/?token=${token}`, {
method: 'POST', method: 'POST',
body: JSON.stringify(event), body: JSON.stringify(event),
headers: { 'Content-Type': 'application/json' } headers: { 'Content-Type': 'application/json' }
});
console.error(146, 'new');
} }
); console.error(148, res.status);
console.log(res.status);
} }
})(); })();
let k = {
__typename: 'Page',
contextual_name: 'Olavshallen',
city: {
cityContextualName: 'Trondheim, Norway',
id: '110429825645017',
contextual_name: 'Trondheim, Norway'
},
id: '149127815110411',
name: 'Olavshallen',
url: 'https://www.facebook.com/olavshallenas/',
__isNode: 'Page'
};

Loading…
Cancel
Save