Browse Source

add gm and commit endpoint

fix-broken-scrape
Jørgen Lien Sellæg 6 years ago
parent
commit
023b9a26ff
  1. 1
      package.json
  2. 14
      scrape.js
  3. 65
      yarn.lock

1
package.json

@ -1,6 +1,7 @@
{
"license": "UNLICENSED",
"dependencies": {
"gm": "^1.23.1",
"jsdom": "^16.2.2",
"puppeteer": "^3.0.2"
}

14
scrape.js

@ -1,10 +1,12 @@
const puppeteer = require('puppeteer');
const { JSDOM } = require('jsdom');
// const event_page = 'https://www.facebook.com/KulturVarsel/events';
const event_page = 'https://www.facebook.com/KulturVarsel/events';
// const event_page = 'https://www.facebook.com/pg/bandmaldito/events';
const event_page = 'file:///home/zalox/src/kultar-events/index.chrome.html';
const upcoming_event_id = 'past_events_card';
const upcoming_event_selector = `#pastup_events_card > div > div:nth-child(2) > table > tbody > tr`;
// const event_page = 'file:///home/zalox/src/kultar-events/index.chrome.html';
const upcoming_event_id = 'upcoming_events_card';
const upcoming_event_selector = `#upcoming_events_card > div > div:nth-child(2) > table > tbody > tr`;
const month_name_to_number = (month_name) => {
switch (month_name.toUpperCase()) {
@ -78,7 +80,6 @@ const parse_ticket_url = (row) => {
const url = new URL(link_text.getElementsByTagName('a')[0].href)
.searchParams
.get('u');
return url;
@ -153,7 +154,7 @@ const load_page = async () => {
.map(parseRowToEvents)
;
console.log(parsed_events);
console.log(JSON.stringify(parsed_events));
}
catch(e) {
@ -164,5 +165,6 @@ const load_page = async () => {
(
async () => {
await load_page();
process.exit();
}
)();

65
yarn.lock

@ -57,6 +57,16 @@ ajv@^6.5.5:
json-schema-traverse "^0.4.1"
uri-js "^4.2.2"
array-parallel@~0.1.3:
version "0.1.3"
resolved "https://registry.yarnpkg.com/array-parallel/-/array-parallel-0.1.3.tgz#8f785308926ed5aa478c47e64d1b334b6c0c947d"
integrity sha1-j3hTCJJu1apHjEfmTRszS2wMlH0=
array-series@~0.1.5:
version "0.1.5"
resolved "https://registry.yarnpkg.com/array-series/-/array-series-0.1.5.tgz#df5d37bfc5c2ef0755e2aa4f92feae7d4b5a972f"
integrity sha1-3103v8XC7wdV4qpPkv6ufUtaly8=
asn1@~0.2.3:
version "0.2.4"
resolved "https://registry.yarnpkg.com/asn1/-/asn1-0.2.4.tgz#8d2475dfab553bb33e77b54e59e880bb8ce23136"
@ -163,6 +173,14 @@ core-util-is@1.0.2:
resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7"
integrity sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=
cross-spawn@^4.0.0:
version "4.0.2"
resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-4.0.2.tgz#7b9247621c23adfdd3856004a823cbe397424d41"
integrity sha1-e5JHYhwjrf3ThWAEqCPL45dCTUE=
dependencies:
lru-cache "^4.0.1"
which "^1.2.9"
cssom@^0.4.4:
version "0.4.4"
resolved "https://registry.yarnpkg.com/cssom/-/cssom-0.4.4.tgz#5a66cf93d2d0b661d80bf6a44fb65f5c2e4e0a10"
@ -203,6 +221,13 @@ debug@4, debug@^4.1.0, debug@^4.1.1:
dependencies:
ms "^2.1.1"
debug@^3.1.0:
version "3.2.6"
resolved "https://registry.yarnpkg.com/debug/-/debug-3.2.6.tgz#e83d17de16d8a7efb7717edbe5fb10135eee629b"
integrity sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==
dependencies:
ms "^2.1.1"
decimal.js@^10.2.0:
version "10.2.0"
resolved "https://registry.yarnpkg.com/decimal.js/-/decimal.js-10.2.0.tgz#39466113a9e036111d02f82489b5fd6b0b5ed231"
@ -365,6 +390,16 @@ glob@^7.1.3:
once "^1.3.0"
path-is-absolute "^1.0.0"
gm@^1.23.1:
version "1.23.1"
resolved "https://registry.yarnpkg.com/gm/-/gm-1.23.1.tgz#2edeeb958084d0f8ea7988e5d995b1c7dfc14777"
integrity sha1-Lt7rlYCE0PjqeYjl2ZWxx9/BR3c=
dependencies:
array-parallel "~0.1.3"
array-series "~0.1.5"
cross-spawn "^4.0.0"
debug "^3.1.0"
har-schema@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/har-schema/-/har-schema-2.0.0.tgz#a94c2224ebcac04782a0d9035521f24735b7ec92"
@ -442,6 +477,11 @@ is-typedarray@~1.0.0:
resolved "https://registry.yarnpkg.com/is-typedarray/-/is-typedarray-1.0.0.tgz#e479c80858df0c1b11ddda6940f96011fcda4a9a"
integrity sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=
isexe@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/isexe/-/isexe-2.0.0.tgz#e8fbf374dc556ff8947a10dcb0572d633f2cfa10"
integrity sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=
isstream@~0.1.2:
version "0.1.2"
resolved "https://registry.yarnpkg.com/isstream/-/isstream-0.1.2.tgz#47e63f7af55afa6f92e1500e690eb8b8529c099a"
@ -527,6 +567,14 @@ lodash@^4.17.15:
resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.15.tgz#b447f6670a0455bbfeedd11392eff330ea097548"
integrity sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==
lru-cache@^4.0.1:
version "4.1.5"
resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-4.1.5.tgz#8bbe50ea85bed59bc9e33dcab8235ee9bcf443cd"
integrity sha512-sWZlbEP2OsHNkXrMl5GYk/jKk70MBng6UU4YI/qGDYbgf6YbP4EvmqISbXCoJiRKs+1bSpFHVgQxvJ17F2li5g==
dependencies:
pseudomap "^1.0.2"
yallist "^2.1.2"
mime-db@1.44.0:
version "1.44.0"
resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.44.0.tgz#fa11c5eb0aca1334b4233cb4d52f10c5a6272f92"
@ -625,6 +673,11 @@ proxy-from-env@^1.0.0:
resolved "https://registry.yarnpkg.com/proxy-from-env/-/proxy-from-env-1.1.0.tgz#e102f16ca355424865755d2c9e8ea4f24d58c3e2"
integrity sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==
pseudomap@^1.0.2:
version "1.0.2"
resolved "https://registry.yarnpkg.com/pseudomap/-/pseudomap-1.0.2.tgz#f052a28da70e618917ef0a8ac34c1ae5a68286b3"
integrity sha1-8FKijacOYYkX7wqKw0wa5aaChrM=
psl@^1.1.28:
version "1.8.0"
resolved "https://registry.yarnpkg.com/psl/-/psl-1.8.0.tgz#9326f8bcfb013adcc005fdff056acce020e51c24"
@ -926,6 +979,13 @@ whatwg-url@^8.0.0:
tr46 "^2.0.0"
webidl-conversions "^5.0.0"
which@^1.2.9:
version "1.3.1"
resolved "https://registry.yarnpkg.com/which/-/which-1.3.1.tgz#a45043d54f5805316da8d62f9f50918d3da70b0a"
integrity sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==
dependencies:
isexe "^2.0.0"
word-wrap@~1.2.3:
version "1.2.3"
resolved "https://registry.yarnpkg.com/word-wrap/-/word-wrap-1.2.3.tgz#610636f6b1f703891bd34771ccb17fb93b47079c"
@ -951,6 +1011,11 @@ xmlchars@^2.2.0:
resolved "https://registry.yarnpkg.com/xmlchars/-/xmlchars-2.2.0.tgz#060fe1bcb7f9c76fe2a17db86a9bc3ab894210cb"
integrity sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==
yallist@^2.1.2:
version "2.1.2"
resolved "https://registry.yarnpkg.com/yallist/-/yallist-2.1.2.tgz#1c11f9218f076089a47dd512f93c6699a6a81d52"
integrity sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI=
yauzl@^2.10.0:
version "2.10.0"
resolved "https://registry.yarnpkg.com/yauzl/-/yauzl-2.10.0.tgz#c7eb17c93e112cb1086fa6d8e51fb0667b79a5f9"

Loading…
Cancel
Save