I want to scrape many websites at once. So, I would prefer to have URL written in the result alongside with the other data that get scrapped. But I don’t know how.
JavaScript
x
26
26
1
const puppeteer = require('puppeteer');
2
3
(async () => {
4
const browser = await puppeteer.launch()
5
const page = await browser.newPage()
6
await page.goto('https://www.amazon.com/')
7
8
9
await page.waitForTimeout( 10000 );
10
const localStorageData = await page.evaluate(() => {
11
let json = {};
12
for (let i = 0; i < localStorage.length; i++) {
13
const key = localStorage.key(i);
14
json[key] = localStorage.getItem(key);
15
}
16
return json;
17
});
18
const data = {};
19
for (let entry of Object.entries(data)) {
20
data[entry.key] = entry.value;
21
}
22
console.log(localStorageData)
23
24
await browser.close()
25
})()
26
Advertisement
Answer
You can simply add the URL you use to your JSON:
JavaScript
1
30
30
1
const puppeteer = require("puppeteer");
2
3
(async () => {
4
const browser = await puppeteer.launch();
5
const page = await browser.newPage();
6
7
const url = "https://www.amazon.com/";
8
await page.goto(url);
9
10
await page.waitForTimeout(10000);
11
const localStorageData = await page.evaluate((url) => {
12
let json = {};
13
for (let i = 0; i < localStorage.length; i++) {
14
const key = localStorage.key(i);
15
json[key] = localStorage.getItem(key);
16
}
17
return json;
18
});
19
localStorageData["url"] = url;
20
const data = {};
21
for (let entry of Object.entries(data)) {
22
data[entry.key] = entry.value;
23
}
24
console.log(localStorageData);
25
26
await browser.close();
27
})();
28
29
30