Scrape data inside a JavaScript function by using python selenium

Tags: , , ,



Please let me know if there is any option available to scrape data inside a JavaScript function below using Selenium

page https://www.bayut.com/index/sale-prices-apartments-abu-dhabi.html

input code

<script type="text/javascript">

var chartData1 = [];
var chartData2 = [];
var chartData3 = [];

function generateChartData() {
            var newDate = new Date('2010-01-31');
            chartData1.push({
            date: newDate,
                    value: 1443,
                    disply_val: '1,443'
            });
            chartData2.push({
            date: newDate,
                    value: 0,
                    disply_val: '0'
            });
            chartData3.push({
            date: newDate,
                    value: 100.0000,
                    disply_val: '100'
            });
                var newDate = new Date('2010-02-28');
            chartData1.push({
            date: newDate,
                    value: 1407,
                    disply_val: '1,407'
            });
            chartData2.push({
            date: newDate,
                    value: 0,
                    disply_val: '0'
            });
            chartData3.push({
            date: newDate,
                    value: 97.5000,
                    disply_val: '98'
            });
                var newDate = new Date('2010-03-31');
            chartData1.push({
            date: newDate,
                    value: 1400,
                    disply_val: '1,400'
            });
            chartData2.push({
            date: newDate,
                    value: 0,
                    disply_val: '0'
            });
            chartData3.push({
            date: newDate,
                    value: 97.0000,
                    disply_val: '97'
            });
                var newDate = new Date('2010-04-30');
            chartData1.push({
            date: newDate,
                    value: 1433,
                    disply_val: '1,433'
            });
            chartData2.push({
            date: newDate,
                    value: 0,
                    disply_val: '0'
            });
            chartData3.push({
            date: newDate,
                    value: 99.3300,
                    disply_val: '99'
            });
                var newDate = new Date('2010-05-31');
            chartData1.push({
            date: newDate,
                    value: 1409,
                    disply_val: '1,409'
            });
            chartData2.push({
            date: newDate,
                    value: 0,
                    disply_val: '0'
            });
            chartData3.push({
            date: newDate,
                    value: 97.6700,
                    disply_val: '98'
            });
                var newDate = new Date('2010-06-30');
            chartData1.push({
            date: newDate,
                    value: 1380,
                    disply_val: '1,380'
            });
            chartData2.push({
            date: newDate,
                    value: 0,
                    disply_val: '0'
            });
            chartData3.push({
            date: newDate,
                    value: 95.6700,
                    disply_val: '96'
            });
                var newDate = new Date('2010-07-31');
            chartData1.push({
            date: newDate,
                    value: 1270,
                    disply_val: '1,270'
            });
            chartData2.push({
            date: newDate,
                    value: 0,
                    disply_val: '0'
            });
            chartData3.push({
            date: newDate,
                    value: 88.0000,
                    disply_val: '88'
            });
                var newDate = new Date('2010-08-31');
            chartData1.push({
            date: newDate,
                    value: 1265,
                    disply_val: '1,265'
            });
            chartData2.push({
            date: newDate,
                    value: 0,
                    disply_val: '0'
            });
            chartData3.push({
            date: newDate,
                    value: 87.6700,
                    disply_val: '88'
            });
                var newDate = new Date('2010-09-30');
            chartData1.push({
            date: newDate,
                    value: 1251,
                    disply_val: '1,251'
            });
            chartData2.push({
            date: newDate,
                    value: 0,
                    disply_val: '0'
            });
            chartData3.push({
            date: newDate,
                    value: 86.6700,
                    disply_val: '87'
            });
                var newDate = new Date('2010-10-31');
            chartData1.push({
            date: newDate,
                    value: 1251,
                    disply_val: '1,251'
            });
            chartData2.push({
            date: newDate,
                    value: 0,
                    disply_val: '0'
            });
            chartData3.push({
            date: newDate,
                    value: 86.6700,
                    disply_val: '87'
            });
                var newDate = new Date('2010-11-30');
            chartData1.push({
            date: newDate,
                    value: 1227,
                    disply_val: '1,227'
            });
            chartData2.push({
            date: newDate,
                    value: 0,
                    disply_val: '0'
            });
            chartData3.push({
            date: newDate,
                    value: 85.0000,
                    disply_val: '85'
            });
                var newDate = new Date('2010-12-31');
            chartData1.push({
            date: newDate,
                    value: 1284,
                    disply_val: '1,284'
            });
            chartData2.push({
            date: newDate,
                    value: 0,
                    disply_val: '0'
            });
            chartData3.push({
            date: newDate,
                    value: 89.0000,
                    disply_val: '89'
            });
                var newDate = new Date('2011-01-31');
            chartData1.push({
            date: newDate,
                    value: 1284,
                    disply_val: '1,284'
            });
            chartData2.push({
            date: newDate,
                    value: 0,
                    disply_val: '0'
            });
            chartData3.push({
            date: newDate,
                    value: 89.0000,
                    disply_val: '89'
            });
                var newDate = new Date('2011-02-28');
            chartData1.push({
            date: newDate,
                    value: 1279,
                    disply_val: '1,279'
            });
            chartData2.push({
            date: newDate,
                    value: 0,
                    disply_val: '0'
            });
            chartData3.push({
            date: newDate,
                    value: 88.6700,
                    disply_val: '89'
            });
                var newDate = new Date('2011-03-31');
            chartData1.push({
            date: newDate,
                    value: 1212,
                    disply_val: '1,212'
            });
            chartData2.push({
            date: newDate,
                    value: 0,
                    disply_val: '0'
            });
            chartData3.push({
            date: newDate,
                    value: 84.0000,
                    disply_val: '84'
            });
                var newDate = new Date('2011-04-30');
            chartData1.push({
            date: newDate,
                    value: 1231,
                    disply_val: '1,231'
            });
            chartData2.push({
            date: newDate,
                    value: 0,
                    disply_val: '0'
            });
            chartData3.push({
            date: newDate,
                    value: 85.3300,
                    disply_val: '85'
            });
                var newDate = new Date('2011-05-31');
            chartData1.push({
            date: newDate,
                    value: 1279,
                    disply_val: '1,279'
            });
            chartData2.push({
            date: newDate,
                    value: 0,
                    disply_val: '0'
            });
            chartData3.push({
            date: newDate,
                    value: 88.6700,
                    disply_val: '89'
            });
                var newDate = new Date('2011-06-30');
            chartData1.push({
            date: newDate,
                    value: 1318,
                    disply_val: '1,318'
            });
            chartData2.push({
            date: newDate,
                    value: 0,
                    disply_val: '0'
            });
            chartData3.push({
            date: newDate,
                    value: 91.3300,
                    disply_val: '91'
            });

Output

Date Value

2010-01-31 1443

2010-02-28 1407

Tried but not working My output file should have date and value columns, please let me know if there is option available

driver.find_element_by_id('newDate')
driver.find_elements_by_xpath("//*[@class=' mask_search']")
 

Answer

use driver.execute_script:

data = driver.execute_script('return [chartData1, chartData2, chartData3]')


Source: stackoverflow