How to make a lot of request requests in Node js?

Question

Node.js

Task:

There are 5000 urls. It is necessary to check each url whether it exists or not. Send a head request and if the response status is 200 then everything is OK.

const axios = require('axios');
const promises = [];

[5000].forEach((url) => {
  promises.push(axios.head(url).catch(() => ({ status: 'error' })));
});
Promise.all(promises)
  .then((results) => {
    //do somthing with result
  })
  .catch((err) => {
    //do somthing with error
  });

I understand that this is not correct, but it works this way, but for a very long time.

Advise where to dig. Give advice. Say that it does not work like this in node js or in general ....
Of course, you can check each url in turn, but this will take a long time.
How to make many requests at the same time and what are the limitations?

Answer 1

For a long time I did about the same task, MB will help.

var http = require('http');
var maxSimRequests = 10; // сколько запросов активно одновременно
var urls = []; // массив ссылок которые надо проверить
var urlsInWork = urls.splice(0, maxSimRequests);
if (urlsInWork.length === 0 || maxSimRequests < 1) return;
urlsInWork.forEach(requestFn);

var voidInterval = setInterval(() => {
  // впринципе можно и без интервала обойтись, не помню зачем его делал
}, 10);

function requestFn(url) {
  http.get(url, (response) => {
    let { statusCode } = response;
    if (statusCode === 200) {
      // что то делаем с response
    } else if (statusCode === 404) {
      urlDone(url);
    } else {
      setTimeout(() => {
        requestFn(url);
      }, 1000);
    };
  }).on('error', (err) => {
    // какойто хендл ошибки
    urlDone(url);
  });
};

function urlDone(url) {
  urlsInWork = urlsInWork.filter(u => u !== url);
  if (urls.length > 0) {
    let nextUrl = urls.pop();
    urlsInWork.push(nextUrl);
    requestFn(nextUrl);
  } else if (urlsInWork.length === 0) {
    clearInterval(voidInterval);
  };
};