this e Node js script is a web crawler using puppeteer + a rotating IP using proxyrack but I need to run this code in phantomjscloud I need each browser to have its own IP address, how can I do that ??
this e Node js script is a web crawler using puppeteer + a rotating IP using proxyrack but I need to run this code in phantomjscloud I need each browser to have its own IP address, how can I do that ??
const puppeteer = require('puppeteer');
const langBtns = require('./langBtns');
const proxy = require('./proxy');
const { promisify } = require('util');
const sleep = promisify(setTimeout);
const devices = Object.values(puppeteer.devices); //Set of mobile devices provided by puppeteer
//Array of all objects found in the langBtns.JSON file
let choices = Object.values(langBtns);
let goodBtnSelector;
let windows = 1000000; //The total number of windows to be opened throughout the execution of the code
let concurrency = 10; /* The number of windows running concurrently.
This variable acts like a semaphore, whenever a window is running, it is decremented;
and whenever a window finishes execution, it is incremented to allow more windows to open */
let flags = [];
flags.fill(true);
let startTime;
let endTime;
let elapsedTime;
let counter = 0;
class BrowserHandler{
constructor(index) {
const launch_browser = async () => {
this.index = index;
this.browser = false;
this.browser = await puppeteer.launch({
headless: false,
slowMo: 10,
executablePath:
"C:\Program Files\BraveSoftware\Brave-Browser\Application\brave.exe",
args: [
`--proxy-server=${proxy.proxyProvider}`,
"--no-sandbox",
"--disable-setuid-sandbox",
"--disable-dev-shm-usage",
"--disable-accelerated-2d-canvas",
"--no-first-run",
"--no-zygote",
"--disable-gpu",
"--disable-notifications"]
});
this.browser.on('disconnected', async() => {
if(this.status === 0){
console.log("Browser ", this.index, " disconnected.");
concurrency++;
doStuff(index).catch(e => {
console.log('Error happened, Oops: ', e);
});
}
});
};
(async () => {
await launch_browser();
console.log("Browser", this.index, "connected.");
})();
}
setStatus(status){
this.status = status;
}
}
const wait_for_browser = browser_handler => new Promise((resolve, reject) => {
const browser_check = setInterval(() => {
if (browser_handler.browser !== false) {
clearInterval(browser_check);
resolve(true);
}
}, 100 );
});
async function start(){
startTime = performance.now();
for (let i = 0; i < windows; i++) {
if(concurrency !== 0){
doStuff(i).catch(e => {
console.log('Error happened, Oops: ', e);
});
}
while(concurrency === 0) {
await sleep(100);
}
}
}
start();
async function checkCounter() {
if (counter === windows) {
endTime = performance.now();
elapsedTime = (endTime - startTime)*0.001; //To seconds
if(elapsedTime < 60){
console.log(counter, " Windows Done in: ", elapsedTime.toPrecision(2), " seconds");
}else if(elapsedTime < 3600){
console.log(counter, " Windows Done in: ", elapsedTime.toPrecision(2) / 60, " minutes");
}else{
console.log(counter, " Windows Done in: ", elapsedTime.toPrecision(3) / 3600, " hours");
}
process.exit();
}
}
async function doStuff(index) {
concurrency--;
const browser_handler = new BrowserHandler(index);
await browser_handler.setStatus(0);
await wait_for_browser(browser_handler);
const [page] = await browser_handler.browser.pages();
await page.setDefaultNavigationTimeout(0);
const randInt = Math.floor(Math.random() * 25001);
if(randInt % 2 === 0){ //Choose mobile
const device = devices[Math.floor(Math.random() * devices.length)];
console.log("");
console.log("Browser ", index, " Device: " , device.name);
await page.emulate(device);
}
else{
console.log("Browser ", index, " Device: Desktop");
}
try{
await page.authenticate({
username: proxy.username,
password: proxy.password,
}).then(async () => {
let randObj = choices[Math.floor(Math.random() * choices.length)];
let lang = randObj.language;
goodBtnSelector = randObj.goodButtonSelector;
await page.goto(`https://coinmarketcap.com/${lang}/`, {"waitUntil":["load", "networkidle2"], timeout: 60000})
.then(async () => {
try {
{
const frame = page.mainFrame();
const promise = page.waitForNavigation();
const element = await frame.waitForSelector(".dQjfsE");
await element.click();
const element2 = await page.waitForSelector(".bzyaeu-3", {timeout: 5000});
await element2.type("Tiger22" , {timeout: 5000});
await element2.press('Enter' , {timeout: 5000});
await promise;
}
{
const frame = page.mainFrame();
await autoScroll(page);
const element = await page.waitForSelector(goodBtnSelector);
await page.waitForTimeout(2000);
await element.click();
await page.waitForTimeout(5000);
await autoScrollUp(page);
console.log(" Browser ", index, "Done ");
await browser_handler.setStatus(1);
await browser_handler.browser.close();
concurrency++;
counter++;
checkCounter();
}
}catch(e){
browser_handler.status = 0;
browser_handler.index = index;
await browser_handler.browser.close();
}
})
})
}catch(e){
browser_handler.status = 0;
browser_handler.index = index;
await browser_handler.browser.close();
}
}
async function autoScroll(page){
await page.evaluate(async () => {
await new Promise((resolve, reject) => {
let totalHeight = 0;
const distance = 100;
const timer = setInterval(() => {
const scrollHeight = document.body.scrollHeight;
window.scrollBy(0, distance);
totalHeight += distance;
if(totalHeight >= scrollHeight){
clearInterval(timer);
resolve();
}
}, 100);
});
});
}
async function autoScrollUp(page){
await page.evaluate(async () => {
await new Promise((resolve, reject) => {
let totalHeight = 0;
const distance = 100;
const timer = setInterval(() => {
const scrollHeight = document.body.scrollHeight;
window.scrollBy(0, -distance);
totalHeight += distance;
if(totalHeight >= scrollHeight){
clearInterval(timer);
resolve();
}
}, 100);
});
});
}
const puppeteer = require('puppeteer');
const langBtns = require('./langBtns');
const proxy = require('./proxy');
const { promisify } = require('util');
const sleep = promisify(setTimeout);
const devices = Object.values(puppeteer.devices); //Set of mobile devices provided by puppeteer
//Array of all objects found in the langBtns.JSON file
let choices = Object.values(langBtns);
let goodBtnSelector;
let windows = 1000000; //The total number of windows to be opened throughout the execution of the code
let concurrency = 10; /* The number of windows running concurrently.
This variable acts like a semaphore, whenever a window is running, it is decremented;
and whenever a window finishes execution, it is incremented to allow more windows to open */
let flags = [];
flags.fill(true);
let startTime;
let endTime;
let elapsedTime;
let counter = 0;
class BrowserHandler{
constructor(index) {
const launch_browser = async () => {
this.index = index;
this.browser = false;
this.browser = await puppeteer.launch({
headless: false,
slowMo: 10,
executablePath:
"C:\Program Files\BraveSoftware\Brave-Browser\Application\brave.exe",
args: [
`--proxy-server=${proxy.proxyProvider}`,
"--no-sandbox",
"--disable-setuid-sandbox",
"--disable-dev-shm-usage",
"--disable-accelerated-2d-canvas",
"--no-first-run",
"--no-zygote",
"--disable-gpu",
"--disable-notifications"]
});
this.browser.on('disconnected', async() => {
if(this.status === 0){
console.log("Browser ", this.index, " disconnected.");
concurrency++;
doStuff(index).catch(e => {
console.log('Error happened, Oops: ', e);
});
}
});
};
(async () => {
await launch_browser();
console.log("Browser", this.index, "connected.");
})();
}
setStatus(status){
this.status = status;
}
}
const wait_for_browser = browser_handler => new Promise((resolve, reject) => {
const browser_check = setInterval(() => {
if (browser_handler.browser !== false) {
clearInterval(browser_check);
resolve(true);
}
}, 100 );
});
async function start(){
startTime = performance.now();
for (let i = 0; i < windows; i++) {
if(concurrency !== 0){
doStuff(i).catch(e => {
console.log('Error happened, Oops: ', e);
});
}
while(concurrency === 0) {
await sleep(100);
}
}
}
start();
async function checkCounter() {
if (counter === windows) {
endTime = performance.now();
elapsedTime = (endTime - startTime)*0.001; //To seconds
if(elapsedTime < 60){
console.log(counter, " Windows Done in: ", elapsedTime.toPrecision(2), " seconds");
}else if(elapsedTime < 3600){
console.log(counter, " Windows Done in: ", elapsedTime.toPrecision(2) / 60, " minutes");
}else{
console.log(counter, " Windows Done in: ", elapsedTime.toPrecision(3) / 3600, " hours");
}
process.exit();
}
}
async function doStuff(index) {
concurrency--;
const browser_handler = new BrowserHandler(index);
await browser_handler.setStatus(0);
await wait_for_browser(browser_handler);
const [page] = await browser_handler.browser.pages();
await page.setDefaultNavigationTimeout(0);
const randInt = Math.floor(Math.random() * 25001);
if(randInt % 2 === 0){ //Choose mobile
const device = devices[Math.floor(Math.random() * devices.length)];
console.log("");
console.log("Browser ", index, " Device: " , device.name);
await page.emulate(device);
}
else{
console.log("Browser ", index, " Device: Desktop");
}
try{
await page.authenticate({
username: proxy.username,
password: proxy.password,
}).then(async () => {
let randObj = choices[Math.floor(Math.random() * choices.length)];
let lang = randObj.language;
goodBtnSelector = randObj.goodButtonSelector;
await page.goto(`https://coinmarketcap.com/${lang}/`, {"waitUntil":["load", "networkidle2"], timeout: 60000})
.then(async () => {
try {
{
const frame = page.mainFrame();
const promise = page.waitForNavigation();
const element = await frame.waitForSelector(".dQjfsE");
await element.click();
const element2 = await page.waitForSelector(".bzyaeu-3", {timeout: 5000});
await element2.type("Tiger22" , {timeout: 5000});
await element2.press('Enter' , {timeout: 5000});
await promise;
}
{
const frame = page.mainFrame();
await autoScroll(page);
const element = await page.waitForSelector(goodBtnSelector);
await page.waitForTimeout(2000);
await element.click();
await page.waitForTimeout(5000);
await autoScrollUp(page);
console.log(" Browser ", index, "Done ");
await browser_handler.setStatus(1);
await browser_handler.browser.close();
concurrency++;
counter++;
checkCounter();
}
}catch(e){
browser_handler.status = 0;
browser_handler.index = index;
await browser_handler.browser.close();
}
})
})
}catch(e){
browser_handler.status = 0;
browser_handler.index = index;
await browser_handler.browser.close();
}
}
async function autoScroll(page){
await page.evaluate(async () => {
await new Promise((resolve, reject) => {
let totalHeight = 0;
const distance = 100;
const timer = setInterval(() => {
const scrollHeight = document.body.scrollHeight;
window.scrollBy(0, distance);
totalHeight += distance;
if(totalHeight >= scrollHeight){
clearInterval(timer);
resolve();
}
}, 100);
});
});
}
async function autoScrollUp(page){
await page.evaluate(async () => {
await new Promise((resolve, reject) => {
let totalHeight = 0;
const distance = 100;
const timer = setInterval(() => {
const scrollHeight = document.body.scrollHeight;
window.scrollBy(0, -distance);
totalHeight += distance;
if(totalHeight >= scrollHeight){
clearInterval(timer);
resolve();
}
}, 100);
});
});
}
