Browse Source

poor mans rotating proxy

fix-broken-scrape
Jørgen Sverre Lien Sellæg 4 years ago
parent
commit
034fb1b251
  1. 20
      src/facebook-request.mjs

20
src/facebook-request.mjs

@ -4,15 +4,29 @@ import * as url from 'url';
import https_proxy_agent from 'https-proxy-agent'; import https_proxy_agent from 'https-proxy-agent';
const proxies = [
'176.112.80.253:12323:14ae0dc1e2459:4364101eab',
'176.112.83.69:12323:14ae0dc1e2459:4364101eab',
'176.112.83.203:12323:14ae0dc1e2459:4364101eab',
'176.112.80.150:12323:14ae0dc1e2459:4364101eab',
'176.112.82.121:12323:14ae0dc1e2459:4364101eab'
].map((proxy_str) => {
const [ip, port, user, password] = proxy_str.split(':');
return { ip, port, user, password };
});
const random_int = (max, min) => Math.floor(Math.random() * (max - min) + min);
export const do_request = async (doc_id, variables, parse = true) => { export const do_request = async (doc_id, variables, parse = true) => {
const params = new URLSearchParams(); const params = new URLSearchParams();
let proxyOpts = url.parse('http://geo.iproyal.com:12323'); const { ip, port, user, password } = proxies[random_int(0, 4)];
proxyOpts.auth = 'zalox:LQq0b7EZzjhjlnN';
let proxyOpts = url.parse(`http://${ip}:${port}`);
proxyOpts.auth = `${user}:${password}`;
const agent = new https_proxy_agent(proxyOpts); const agent = new https_proxy_agent(proxyOpts);
params.append('doc_id', doc_id); params.append('doc_id', doc_id);
params.append('variables', JSON.stringify(variables)); params.append('variables', JSON.stringify(variables));
console.log(variables);
const fetch_options = { const fetch_options = {
headers: { headers: {
'Content-Type': 'application/x-www-form-urlencoded' 'Content-Type': 'application/x-www-form-urlencoded'

Loading…
Cancel
Save