So I am trying to use a proxy to scrape json data from my api site using some code here:
request({
url: `https://gist.githubusercontent.com/im-moies/d9ab609225221fccd5bfc7883d43090d/raw/d865afe7deb50d41168f0dde0302f3a9cc700025/gistfile1.txt`,
proxy: proxyGenerator(),
json: true,
}, (err, res, body) => {
if (err) {
return console.log(err);
}
but the proxyGenerator() doesn't work which I'm using the exact code from https://zenscrape.com/how-to-build-a-simple-proxy-rotator-in-node-js/
(code works theres more just the proxy thing doesn't seem to work with the json for some reason?)
function proxyGenerator() {
let ip_addresses = [];
let port_numbers = [];
let proxy;
request("https://sslproxies.org/", function(error, response, html) {
if (!error && response.statusCode == 200) {
const $ = cheerio.load(html);
$("td:nth-child(1)").each(function(index, value) {
ip_addresses[index] = $(this).text();
});
$("td:nth-child(2)").each(function(index, value) {
port_numbers[index] = $(this).text();
});
} else {
console.log("Error loading proxy, please try again");
}
ip_addresses.join(", ");
port_numbers.join(", ");