Hi everyone,
I’m playing around with Node.js and cheerio package as part of my node.js learning and im trying to build a web scrapper that will get the title and the price of an item from a shopping site but when I try to console.log the html variable it returns a basic html structure with some Js functions that are trying to prevent the scraping.
my code:
JavaScript
x
23
23
1
const needle = require('needle')2
const http = require('http')3
const cheerio = require("cheerio")4
5
needle.get('https://ksp.co.il/web/item/130984', (error, response, html) => {6
if (!error && response.statusCode == 200){7
const $ = cheerio.load(html)8
9
10
11
12
console.log(html)13
14
http.createServer(function (req, res) {15
res.writeHead(200, {'Content-Type': 'text/html'});16
res.write(html)17
res.end();18
}).listen(3000);19
20
21
}22
})23
I guess it’s some kind of protection layer from scrapers but this what i get as a result:
JavaScript
1
176
176
1
<html lang="he">2
3
<head>4
<meta charset="utf-8" />5
<link rel="icon" id="header-icon" href="/web/favicon.ico">6
<link rel="canonical" id="header-canonical">7
<meta name="viewport" content="width=device-width,initial-scale=1" />8
<meta name="description" content="מעל 38,000 מוצרים: מחשבים סלולר, בשמים, למטבח, למשרד טיפוח, פארם, צעצועים, נעלים ומיזוג" />9
<link rel="manifest" href="/web/manifest.json" />10
<script src='https://ksp.co.il/_cache/dictionary_site_only/all.js?ts=1640275687'></script>11
<script src="/web/encoding.min.js"></script>12
<script>13
! function() {14
if ("function" == typeof window.CustomEvent) return !1;15
16
function t(t, e) {17
e = e || {18
bubbles: !1,19
cancelable: !1,20
detail: void 021
};22
var n = document.createEvent("CustomEvent");23
return n.initCustomEvent(t, e.bubbles, e.cancelable, e.detail), n24
}25
t.prototype = window.Event.prototype, window.CustomEvent = t26
}()27
28
29
30
</script>31
<script src='https://ksp.co.il/_cache/menu_dev/menu.js?ts=1640280140'></script>32
<style>33
.lang_he svg.revert {34
transform: scaleX(-1)35
}36
37
* {38
box-sizing: border-box39
}40
41
a:visited {42
color: inherit43
}44
45
</style>46
<title>KSP</title>47
<link rel="preload" href="/web/fonts/Assistant/w300en.woff2" as="font" crossorigin="anonymous">48
<link rel="preload" href="/web/fonts/Assistant/w400en.woff2" as="font" crossorigin="anonymous">49
<link rel="preload" href="/web/fonts/Assistant/w600en.woff2" as="font" crossorigin="anonymous">50
<link rel="preload" href="/web/fonts/Assistant/w700en.woff2" as="font" crossorigin="anonymous">51
<link rel="preload" href="/web/fonts/Assistant/w800en.woff2" as="font" crossorigin="anonymous">52
<link rel="preload" href="/web/fonts/Assistant/w300he.woff2" as="font" crossorigin="anonymous">53
<link rel="preload" href="/web/fonts/Assistant/w400he.woff2" as="font" crossorigin="anonymous">54
<link rel="preload" href="/web/fonts/Assistant/w600he.woff2" as="font" crossorigin="anonymous">55
<link rel="preload" href="/web/fonts/Assistant/w700he.woff2" as="font" crossorigin="anonymous">56
<link rel="preload" href="/web/fonts/Assistant/w800he.woff2" as="font" crossorigin="anonymous">57
<link rel="stylesheet" href="/web/fonts/Assistant/index.css">58
<script>59
! function(e, t, a, n, g) {60
e[n] = e[n] || [], e[n].push({61
"gtm.start": (new Date).getTime(),62
event: "gtm.js"63
});64
var m = t.getElementsByTagName(a)[0],65
r = t.createElement(a);66
r.async = !0, r.src = "https://www.googletagmanager.com/gtm.js?id=GTM-59D9ZCV", m.parentNode.insertBefore(r, m)67
}(window, document, "script", "dataLayer")68
69
</script>70
<link href="/web/static/css/2.f825bd5a.chunk.css" rel="stylesheet">71
<link href="/web/static/css/main.4e4460ee.chunk.css" rel="stylesheet">72
</head>73
74
<body style="overflow-x:hidden"><noscript><iframe src="https://www.googletagmanager.com/ns.html?id=GTM-59D9ZCV" height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript><noscript>You need to enable JavaScript to run this app.</noscript>75
<div>76
<div id="root"></div>77
</div>78
<script async src="https://www.googletagmanager.com/gtag/js?id=UA-109261-1"></script>79
<script async src="https://www.googletagmanager.com/gtag/js?id=AW-1032006858"></script>80
<script>81
function gtag() {82
dataLayer.push(arguments)83
}84
window.dataLayer = window.dataLayer || [], gtag("js", new Date)85
86
</script>87
<script>88
! function(e, t, n, c, o, a, f) {89
e.fbq || (o = e.fbq = function() {90
o.callMethod ? o.callMethod.apply(o, arguments) : o.queue.push(arguments)91
}, e._fbq || (e._fbq = o), o.push = o, o.loaded = !0, o.version = "2.0", o.queue = [], (a = t.createElement(n)).async = !0, a.src = "https://connect.facebook.net/en_US/fbevents.js", (f = t.getElementsByTagName(n)[0]).parentNode.insertBefore(a, f))92
}(window, document, "script"), fbq("init", "1179615532183839"), fbq("track", "PageView")93
94
</script><noscript><img height="1" width="1" style="display:none" src="https://www.facebook.com/tr?id=1179615532183839&ev=PageView&noscript=1" /></noscript>95
<script>96
! function(e) {97
function r(r) {98
for (var n, l, f = r[0], i = r[1], a = r[2], p = 0, s = []; p < f.length; p++) l = f[p], Object.prototype.hasOwnProperty.call(o, l) && o[l] && s.push(o[l][0]), o[l] = 0;99
for (n in i) Object.prototype.hasOwnProperty.call(i, n) && (e[n] = i[n]);100
for (c && c(r); s.length;) s.shift()();101
return u.push.apply(u, a || []), t()102
}103
104
function t() {105
for (var e, r = 0; r < u.length; r++) {106
for (var t = u[r], n = !0, f = 1; f < t.length; f++) {107
var i = t[f];108
0 !== o[i] && (n = !1)109
}110
n && (u.splice(r--, 1), e = l(l.s = t[0]))111
}112
return e113
}114
var n = {},115
o = {116
1: 0117
},118
u = [];119
120
function l(r) {121
if (n[r]) return n[r].exports;122
var t = n[r] = {123
i: r,124
l: !1,125
exports: {}126
};127
return e[r].call(t.exports, t, t.exports, l), t.l = !0, t.exports128
}129
l.m = e, l.c = n, l.d = function(e, r, t) {130
l.o(e, r) || Object.defineProperty(e, r, {131
enumerable: !0,132
get: t133
})134
}, l.r = function(e) {135
"undefined" != typeof Symbol && Symbol.toStringTag && Object.defineProperty(e, Symbol.toStringTag, {136
value: "Module"137
}), Object.defineProperty(e, "__esModule", {138
value: !0139
})140
}, l.t = function(e, r) {141
if (1 & r && (e = l(e)), 8 & r) return e;142
if (4 & r && "object" == typeof e && e && e.__esModule) return e;143
var t = Object.create(null);144
if (l.r(t), Object.defineProperty(t, "default", {145
enumerable: !0,146
value: e147
}), 2 & r && "string" != typeof e)148
for (var n in e) l.d(t, n, function(r) {149
return e[r]150
}.bind(null, n));151
return t152
}, l.n = function(e) {153
var r = e && e.__esModule ? function() {154
return e.default155
} : function() {156
return e157
};158
return l.d(r, "a", r), r159
}, l.o = function(e, r) {160
return Object.prototype.hasOwnProperty.call(e, r)161
}, l.p = "/web/";162
var f = this.webpackJsonpcode = this.webpackJsonpcode || [],163
i = f.push.bind(f);164
f.push = r, f = f.slice();165
for (var a = 0; a < f.length; a++) r(f[a]);166
var c = i;167
t()168
}([])169
170
</script>171
<script src="/web/static/js/2.159a3d73.chunk.js"></script>172
<script src="/web/static/js/main.1ab08410.chunk.js"></script>173
</body>174
175
</html>176
Any idea how can i overcome this ? Thanks everyone
Advertisement
Answer
This likely is not scraper protection. Instead, this site is probably using some web framework that loads in the viewable data and DOM elements after the JS has run. The easiest way to get past this would be to use a library like puppeteer that will load the site and process it like how a real browser would. Here is a basic example of what you might want:
JavaScript
1
26
26
1
const puppeteer = require('puppeteer');2
3
(async () => {4
const browser = await puppeteer.launch();5
const page = await browser.newPage();6
await page.goto('[the full URL you want to scrape]');7
8
// once the page has loaded, you can find data in a few ways:9
10
// 1: querying11
const elements = await page.$$("[any JS selector]")12
13
// 2: evaluate14
const elements1 = await page.evaluate(() => {15
// run any code on the site and have it's result returned to you16
});17
18
// 3: text19
const wholePage = await page.evaluate(() => document.querySelector("*").outerHTML);20
21
// this gives you the text content of the whole page22
// which you can then put in to cheerio or any parser23
// and use how you were using before24
25
await browser.close();26
})();You can read more about puppeteer more broadly, method 1, method 2 and method 3.