Browse Source

move function to browse file

fix-broken-scrape
Jørgen Lien Sellæg 6 years ago
parent
commit
8bd1075285
  1. 4
      src/browse.js
  2. 6
      src/logic.js

4
src/browse.js

@ -2,7 +2,6 @@ const puppeteer = require('puppeteer');
const { const {
has_past_events, has_past_events,
get_body_inner_text,
has_upcoming_events, has_upcoming_events,
get_upcoming_events_from_page, get_upcoming_events_from_page,
get_past_events_from_page, get_past_events_from_page,
@ -47,6 +46,9 @@ const register_page_scraper = (endpoint, page, past_events = false) => {
}); });
}; };
const get_body_inner_text = async (page) =>
await page.evaluate('document.querySelector("body").innerText;');
const get_page_events = async (opt) => { const get_page_events = async (opt) => {
const browser = await open_browser(opt); const browser = await open_browser(opt);
const facebook_page = await browser.newPage(); const facebook_page = await browser.newPage();

6
src/logic.js

@ -1,4 +1,4 @@
const { hasPath, pathOr, props, uniqBy, eqBy, prop, union } = require('ramda'); const { hasPath, pathOr, props } = require('ramda');
const parseArgs = require('minimist'); const parseArgs = require('minimist');
const process = require('process'); const process = require('process');
@ -125,9 +125,6 @@ const map_event = ({ node: event }) => {
}; };
}; };
const get_body_inner_text = async (page) =>
await page.evaluate('document.querySelector("body").innerText;');
const has_upcoming_events = (body) => const has_upcoming_events = (body) =>
body.includes('upcoming events') && body.includes('upcoming events') &&
!body.includes('not have any upcoming events'); !body.includes('not have any upcoming events');
@ -163,7 +160,6 @@ const event_date_to_date_obj = (event) => {
module.exports = { module.exports = {
by_date, by_date,
event_date_to_date_obj, event_date_to_date_obj,
get_body_inner_text,
get_past_events_from_page, get_past_events_from_page,
get_upcoming_events_from_page, get_upcoming_events_from_page,
has_past_events, has_past_events,

Loading…
Cancel
Save