Browse Source

add tools

fix-broken-scrape
Jørgen Sverre Lien Sellæg 4 years ago
parent
commit
02f4443592
  1. 53
      bin/adapter.mjs
  2. 23
      bin/count.mjs
  3. 76
      bin/old.mjs
  4. 10
      bin/request.mjs

53
bin/adapter.mjs

@ -0,0 +1,53 @@
import fetch from 'node-fetch';
import { promises as fs } from 'fs';
const eventFiles = process.argv.slice(3);
const later = (delay, value) =>
new Promise((resolve) => setTimeout(resolve, delay, value));
(async () => {
const root = '/home/zalox/src/acne/lib/derma';
let requests = [];
const promises = eventFiles.map(async (filename) => {
const path = `${root}/${filename}`;
const file_content = await fs.readFile(path, { encoding: 'utf-8' });
const j = JSON.parse(file_content);
for (let i = 0; i < j.length; i++) {
let event = j[i];
const payload = {
canceled: false,
date: {
created: new Date(),
end: '',
start: new Date(event.time_range.start),
updated: new Date()
},
draft: false,
facebook_id: event.id,
host: event.event_place?.name ?? '',
location: event.event_place?.city?.name ?? 'Trondheim, Norway',
name: event.name ?? 'NAME_MISSING',
ticket_url: event.event_buy_ticket_url ?? ''
};
requests.push(payload);
}
});
await Promise.all(promises);
for (let i = 0; i < requests.length; i++) {
let event = requests[i];
const res = fetch(
'http://localhost:3333/events?token=831411806230c7e950c4eeb226499ef92bb6bdc4157797929a0e16d133dc13a8',
{
method: 'POST',
body: JSON.stringify(event),
headers: { 'Content-Type': 'application/json' }
}
);
// console.log(res.status);
}
})();

23
bin/count.mjs

@ -0,0 +1,23 @@
import { promises as fs } from 'fs';
const eventFiles = process.argv.slice(3);
(async () => {
const root = '/home/zalox/src/acne/lib/derma';
let requests = 0;
const promises = eventFiles.map(async (filename) => {
const path = `${root}/${filename}`;
const file_content = await fs.readFile(path, { encoding: 'utf-8' });
const j = JSON.parse(file_content);
console.log(path);
return j.length;
});
for (let i = 0; i < promises.length; i++) {
const l = await promises[i];
requests += l;
}
console.log(requests);
})();

76
bin/old.mjs

@ -0,0 +1,76 @@
import { promises as fs } from 'fs';
import events from './events.json' assert { type: 'json' };
import fetch from 'node-fetch';
import { promises as fs } from 'fs';
const eventFiles = process.argv.slice(3);
const later = (delay, value) =>
new Promise((resolve) => setTimeout(resolve, delay, value));
(async () => {
const root = '/home/zalox/src/acne/lib/derma';
let requests = [];
const promises = eventFiles.map(async (filename) => {
const path = `${root}/${filename}`;
const file_content = await fs.readFile(path, { encoding: 'utf-8' });
const j = JSON.parse(file_content);
for (let i = 0; i < j.length; i++) {
let event = j[i];
const payload = {
canceled: false,
date: {
created: new Date(),
end: '',
start: new Date(event.time_range.start),
updated: new Date()
},
draft: false,
facebook_id: event.id,
host: event.event_place?.name ?? '',
location: event.event_place?.city?.name ?? 'Trondheim, Norway',
name: event.name ?? 'NAME_MISSING',
ticket_url: event.event_buy_ticket_url ?? ''
};
requests.push(payload);
}
});
await Promise.all(promises);
for (let i = 0; i < requests.length; i++) {
let event = requests[i];
const res = await fetch(
'http://localhost:3333/events?token=831411806230c7e950c4eeb226499ef92bb6bdc4157797929a0e16d133dc13a8',
{
method: 'POST',
body: JSON.stringify(event),
headers: { 'Content-Type': 'application/json' }
}
);
}
})();
(async () => {
const promises = events.map(async (filename) => {
const path = `${root}/${filename}`;
const file_content = await fs.readFile(path, { encoding: 'utf-8' });
const j = JSON.parse(file_content);
console.log(path);
return j.length;
});
for (let i = 0; i < promises.length; i++) {
const l = await promises[i];
requests += l;
}
console.log(requests);
})();

10
bin/request.mjs

@ -0,0 +1,10 @@
import fetch from 'node-fetch';
(async () => {
let iterations = 1000;
let start = new Date().valueOf();
for (let i = 0; i < iterations; i++) {
await fetch('http://localhost:3333/events/');
}
let end = new Date().valueOf();
console.log((end - start) / 1000);
})();
Loading…
Cancel
Save