I have built a simple scraper with Puppeteer which I can run locally on my machine, but when I deploy it as a Google Cloud function, it’s not working. The only error message I get from the Google Cloud Logs is:
Function failed on loading user code. This is likely due to a bug in
the user code.
Here are the steps I follow to deploy it the function; code is further below.
(Note: I’m outlining the process of zipping the files; I have tried the Cloud Function inline editor as well but am receiving the same error.)
- Run npm install
- Run zip -r my-app.zip *
- Create new Google Cloud function
— Name ‘newFunction’
— Memory: 1gb
— Runtime: Node.js 14
— Entry point: scrapeFunction - Upload Zip
index.js
const puppeteer = require('puppeteer');
const { BigQuery } = require('@google-cloud/bigquery');
async function scrapeFunction() {
const browser = await puppeteer.launch({ args: ['--no-sandbox', '--disable-setuid-sandbox'] });
const page = await browser.newPage();
await page.goto('<URL>', {waitUntil: 'load', timeout: 0});
await page.waitForSelector('span.text');
const info = await page.evaluate(() => {
return document.querySelector('span.text').innerText;
});
console.log(info);
// Write results to BigQuery table
const bigqueryClient = new BigQuery();
const dataset = bigqueryClient.dataset('xxx');
const table = dataset.table('yyy');
const rows = [{ info: info }];
await table.insert(rows);
await browser.close();
}
scrapeFunction();
package.json
{
"name": "newFunction",
"version": "1.0.0",
"description": "",
"main": "index.js",
"scripts": {
"test": "echo "Error: no test specified" && exit 1"
},
"keywords": [],
"author": "",
"license": "ISC",
"dependencies": {
"@google-cloud/bigquery": "^6.1.0",
"puppeteer": "^19.7.1"
}
}