M
M
Maxim Supreme2018-03-15 14:33:55
Node.js
Maxim Supreme, 2018-03-15 14:33:55

How to collect a list of article urls from a website into an array?

It is necessary to collect a list of urls from the donor site and save it to an array that will be further parsed.
I understand that let aUrl = "there should be a loop that will run through all the urls of the site and collect them in an array"
The rest works as it should!

const tress = require('tress');
const needle = require("needle");
const cheerio = require("cheerio");
const async = require("async");
const fs = require('fs');
const Json2csvParser = require('json2csv').Parser;

let aUrl = ??????

const jquery = body => cheerio.load(body);

let products = [];

let parsePage = ($) => {
    let name = $("#shop-production-view > h1").first().text();

    let categories = $(".breadcrumb").text();
    // let arrCategories = categories.split(' / ');

    let price = $(".price").text();
    let content = $('.content_item').html();
    let images = $(".image").find("img").attr("src");

    // let $imageLink = $(".shop-production-view .image a"),
    //     img = '';
    // if ($imageLink.length > 0) {
    //     img = $imageLink.attr("href");
    // }


    products.push({
        name,
        categories,
        // arrCategories,
        content,
        price,
        images,
        // img,
    });
};

let q = tress((url, callback) => {
    needle.get(url, {  }, (err, res) => {
        if (err) {
            throw err;
        }

        parsePage(jquery(res.body));

        callback();
    });
}, 1);

q.drain = () => {
    // console.log(products);

    fs.writeFile('data.json', JSON.stringify(products), (err) => {
        if (err) throw err;

        console.log("saved");
    });

};

for (let i = 0; i < aUrl.length; i++) {
    q.push(aUrl[i]);
}

Answer the question

In order to leave comments, you need to log in

7 answer(s)
D
Dmitry Eremin, 2018-03-15
@EreminD

well, apparently
let aUrl $(".a").attr("href")
... no?

K
kolkasnf, 2013-11-14
@IgnatVoronov

When turning on the PC, hold down the "Delete" button

K
kolkasnf, 2013-11-14
@kolkasnf

one of the function keys F1-F12. You need to look at the manufacturer's website.

A
ArtKuz, 2013-11-14
@ArtKuz

Most often the "Del", "Tab" or "F2" key

D
DestroyKun, 2013-11-14
@DestroyKun

Delete, written by the manufacturer.

V
Valentine, 2013-11-14
@ProFfeSsoRr

In Windows 8, you can reboot into the BIOS from Windows: mouse to the right - Change PC Settings - Update and recovery - there is an Advanced startup with a Restart button - press it and get into "safe mode" after reboot, there Troubleshoot - Advanced - UEFI - button again Restart and here it is, UEFI! For example, there is a video here: http://www.youtube.com/watch?v=nsp8XFcAS7o , watch from 1:25

V
Valentine, 2013-11-14
@ProFfeSsoRr

It's just a fast loading of the BIOS, on relatively old gigabyte mothers with a full fast boot, you can't get into the BIOS at all - it's just that usb devices are not initialized by it :)

Didn't find what you were looking for?

Ask your question

Ask a Question

731 491 924 answers to any question