I’m doing a Discord Bot and I have a infinite loop with setInterval each 10s but every loop that the setInterval does, it gives me every data of each loop, so I’d like to know how can I do to get only the last data of the last cycle, not every one.
const puppeteer = require('puppeteer');
const Discord = require('discord.js');
const client = new Discord.Client();
const url = 'url to scrape';
var clocks = [];
(async () => {
const URL = url
const browser = await puppeteer.launch()
const page = await browser.newPage()
await page.goto(URL, { 'waitUntil' : 'networkidle2' });
setInterval(async () => {
let clock = await page.evaluate(()=>{
var a = document.getElementById("task-listing-datatable").getAttribute("data-tasks");
var ar = eval(a);
var keyword = ['asdad', 'asdakdada', 'mama', 'Duplicate Fashion Product Identification Task'];
for(let i=0; i<ar.length; i++){
for(let j=0; j<keyword.length; j++){
if(ar[i][1] === keyword[j]){
let job = (`${ar[i][1]}`);
return (`${ar[i][0]} ${ar[i][1]} Paga ${ar[i][3]} Tareas: ${ar[i][5]}`);
}
}
}
});
console.log(`==== first login ====`)
console.log(`==================`)
if(!clocks.includes(clock)) {
client.on('message', (message)=>{
if(message.author.bot === false) {
message.channel.send(clock);
}
});
clocks.push(clock);
// Save the clock so you will remember it next time.
}
await page.reload();
}, 8000)
})()
client.login('discordjs token');
This is how the messages are shown:
As you can see, now it’s giving each change not all the data of each cycle enter image description here
Advertisement
Answer
Every time your setInterval runs, it loads the page fresh, gathers information in ‘clock’, and sends it via discord. The problem is, it does not know what it has already sent you, so you’ll get some of the same data every time.
The solution to that is to save the data it finds, and then only create a discord message if the current batch of data is different from all of the previous data.
So you want some kind of data store:
var clocks = [];
(async () => {
setInterval(async () => {
const URL = url
const browser = await puppeteer.launch()
// ...
And then once you’ve gotten the current clock back, you want to check if it is NOT in the data store.
if(!clocks.includes(clock)) {
If it isn’t, then you know that you have a new piece of data to send.
if(!clocks.includes(clock)) {
client.on('message', (message)=>{
message.channel.send(clock);
});
clocks.push(clock); // Save the clock so you will remember it next time.
}
So all in all you have something like:
var clocks = [];
(async () => {
setInterval(async () => {
const URL = url
const browser = await puppeteer.launch()
const page = await browser.newPage()
await page.goto(URL, { 'waitUntil' : 'networkidle2' })
let clock = await page.evaluate(()=>{
var a = document.getElementById("task-listing-datatable").getAttribute("data-tasks");
var ar = eval(a);
var keyword = ['asdad', 'asdakdada', 'mama', 'What Is The Best Dialogue Category About Phones'];
for(let i=0; i<ar.length; i++){
for(let j=0; j<keyword.length; j++){
if(ar[i][1] === keyword[j]){
let job = (`${ar[i][1]}`);
return (`${ar[i][0]} ${ar[i][1]} Paga ${ar[i][3]} Tareas: ${ar[i][5]}`);
}
}
}
});
console.log(`==== first login ====`)
console.log(`==================`)
if(!clocks.includes(clock)) {
client.on('message', (message)=>{
message.channel.send(clock);
});
clocks.push(clock); // Save the clock so you will remember it next time.
}
await page.reload();
console.log(`after reload`)
}, 8000)
})()
While we’re at it though, there’s no real reason to fire up a new browser window every 10 seconds, it will probably be easier on your computer to load the page once and then simply refresh every 10 seconds.
var clocks = [];
(async () => {
const URL = url
const browser = await puppeteer.launch()
const page = await browser.newPage()
await page.goto(URL, { 'waitUntil' : 'networkidle2' });
setInterval(async () => {
let clock = await page.evaluate(()=>{
var a = document.getElementById("task-listing-datatable").getAttribute("data-tasks");
var ar = eval(a);
var keyword = ['asdad', 'asdakdada', 'mama', 'What Is The Best Dialogue Category About Phones'];
for(let i=0; i<ar.length; i++){
for(let j=0; j<keyword.length; j++){
if(ar[i][1] === keyword[j]){
let job = (`${ar[i][1]}`);
return (`${ar[i][0]} ${ar[i][1]} Paga ${ar[i][3]} Tareas: ${ar[i][5]}`);
}
}
}
});
console.log(`==== first login ====`)
console.log(`==================`)
if(!clocks.includes(clock)) {
client.on('message', (message)=>{
message.channel.send(clock);
});
clocks.push(clock); // Save the clock so you will remember it next time.
}
await page.reload();
}, 8000)
})()
Now, to make sure that your page function (clock) finds a new data point each time, we need to pass our past data points in to it:
let clock = await page.evaluate(clocks=>{
// ...
}, clocks);
Now, inside of the page function you’ll have access to the old data points.
Instead of
if(ar[i][1] === keyword[j]){
let job = (`${ar[i][1]}`); // What is this for?
return (`${ar[i][0]} ${ar[i][1]} Paga ${ar[i][3]} Tareas: ${ar[i][5]}`);
}
Check if the data point exists in your clocks array, and only return it if it’s new.
if(ar[i][1] === keyword[j]){
let dataPoint =`${ar[i][0]} ${ar[i][1]} Paga ${ar[i][3]} Tareas: ${ar[i][5]}`;
if(!clocks.includes(dataPoint)){
return dataPoint;
}
}