I am trying to create a small crawler as a chrome extension. How it works is:
Open new window or tab.
Perform a search for Google / Google News / YouTube with given keywords.
Store information of the results in a small database
I first created and tested the functions with a popup.html. There it works perfectly.You click on a button and all pages are visited and the results are stored in a database. But I want to start the program without clicking anything first. That’s why I migrated it to background.js. There it also works, but only if the Service Worker / DevTool console is open. Only then it runs completely.
I would be grateful for any helpful answer.
const keywords = [
"Keyword1",
"Keyword2",
// ...
"Keyword13"
];
chrome.runtime.onStartup.addListener(() => {
chrome.tabs.onUpdated.addListener(loadingWindow);
openWindow();
});
// Opens new Window or Tab with the correct URL
function openWindow() {
chrome.tabs.onUpdated.addListener(loadingWindow);
if (runs == 0) {
chrome.windows.create({ url: getUrl(keywords[runs]), type: "normal" }, newWindow => {
window_id = newWindow.id;
});
} else {
chrome.tabs.update(tab_id, { url: getUrl(keywords[runs]) });
}
}
// Wait to load the new tab
function loadingWindow(tabId, changeInfo, tab) {
if (changeInfo.status === 'complete' && tab.status == 'complete' && tab.windowId == window_id) {
tab_id = tabId;
console.log(tab.windowId);
chrome.tabs.onUpdated.removeListener(loadingWindow);
chrome.tabs.sendMessage(tab.id, { text: source }, doStuffWithDom);
}
};
// Get information from content script -> payload and then send to database
function doStuffWithDom(domContent) {
let payload = {... }
var data = new FormData();
data.append("json", JSON.stringify(payload));
fetch(".../store.php", { method: "POST", body: data });
crawlDone();
}
// open new window / tab or close the open window
function crawlDone() {
runs++;
if (runs < keywords.length) {
openWindow();
} else if (runs == keywords.length) {
chrome.windows.remove(window_id);
}
};