Browse Source

create more tools to make my life easier

fix-broken-scrape
Jørgen Sverre Lien Sellæg 5 years ago
parent
commit
b04cdbe8ec
  1. 16
      bin/extract-unique-pages-from-event-nodes
  2. 28
      bin/filter-by-ids
  3. 7
      bin/orland-pages-from-event-nodes
  4. 23
      bin/remove-profile-picture

16
bin/extract-unique-pages-from-event-nodes

@ -0,0 +1,16 @@
#!/usr/bin/env bash
PATH="$PATH:$HOME/src/garbit/lib/facebook-event-scraper/bin"
if [ "$1" == "" ]; then
echo "This script takes in an path to an JSON file containing an Array.";
exit 0;
fi
cat $1 \
| get-hosts-from-event-nodes \
| flatten-array \
| get-pages-from-hosts \
| unique-by-id \
| remove-profile-picture

28
bin/filter-by-ids

@ -0,0 +1,28 @@
#!/usr/bin/env node
const pathOr = require('ramda/src/pathOr');
let input = [];
// YOLO
let ids = eval(process.argv.slice(2).pop()).map((id) => `${id}`);
process.stdin.resume();
process.stdin.setEncoding('utf8');
process.stdin.on('data', (data) => {
input.push(data);
});
process.stdin.on('end', () => {
const str = input.join('');
const arr = JSON.parse(str);
const output = arr.reduce((acc, elem) => {
if (ids.includes(elem.id)) {
return acc;
} else {
return [...acc, elem];
}
}, []);
console.log(JSON.stringify(output));
});

7
bin/orland-pages-from-event-nodes

@ -0,0 +1,7 @@
#!/usr/bin/env bash
PATH="$PATH:$HOME/src/garbit/lib/facebook-event-scraper/bin"
extract-unique-pages-from-event-nodes "$1" \
| filter-by-ids \
"[119117068112852,170097818488]"

23
bin/remove-profile-picture

@ -0,0 +1,23 @@
#!/usr/bin/env node
const pathOr = require('ramda/src/pathOr');
let input = [];
process.stdin.resume();
process.stdin.setEncoding('utf8');
process.stdin.on('data', (data) => {
input.push(data);
});
process.stdin.on('end', () => {
const str = input.join('');
const events = JSON.parse(str);
let uniquePages = [];
console.log(
JSON.stringify(events.map(({ profilePicture, ...event }) => event)),
);
});
Loading…
Cancel
Save