M
M
Maxim Supreme2018-03-14 17:21:27
Node.js
Maxim Supreme, 2018-03-14 17:21:27

How to delay execution?

You need to make a delay so that the server does not drop the connection due to many requests

const tress = require('tress');
const needle = require("needle");
const cheerio = require("cheerio");
const async = require("async");
const fs = require('fs');
const Json2csvParser = require('json2csv').Parser;

let aUrl = [
    'ссылка',
    'ссылка',
    'ссылка',
]

const jquery = body => cheerio.load(body);

let products = [];

let parsePage = ($) => {
    let name = $("#shop-production-view > h1").first().text();

    let categories = $(".breadcrumb").text();
    // let arrCategories = categories.split(' / ');

    let price = $(".price").text();
    let content = $('.content_item').html();
    let images = $(".image").find("img").attr("src");

    // let $imageLink = $(".shop-production-view .image a"),
    //     img = '';
    // if ($imageLink.length > 0) {
    //     img = $imageLink.attr("href");
    // }


    products.push({
        name,
        categories,
        // arrCategories,
        content,
        price,
        images,
        // img,
    });
};

let q = tress((url, callback) => {
    needle.get(url, {  }, (err, res) => {
        if (err) {
            throw err;
        }

        parsePage(jquery(res.body));

        callback();
    });
}, 1);

q.drain = () => {
    // console.log(products);

    fs.writeFile('data.json', JSON.stringify(products), (err) => {
        if (err) throw err;

        console.log("saved");
    });

};

for (let i = 0; i < aUrl.length; i++) {
    q.push(aUrl[i]);
}

Answer the question

In order to leave comments, you need to log in

2 answer(s)
A
Andrey Tsvetkov, 2018-03-14
@MSupreme

let q = tress((url, callback) => {
    needle.get(url, {  }, (err, res) => {
        if (err) {
            throw err;
        }

        parsePage(jquery(res.body));

        setTimeout(callback, 1000);
    });
}, 1);

V
Vladimir Skibin, 2018-03-14
@megafax

Put setTimeout on the next GET and that's it

Didn't find what you were looking for?

Ask your question

Ask a Question

731 491 924 answers to any question