What algorithm of actions is better? How to send a POST request using the request module? How to save an array of data on the server and update it once a day?

dmitriu2562020-04-30 13:12:58

JavaScript

dmitriu256, 2020-04-30 13:12:58

How to do the right thing?
The task is to get a list of company cities from a third-party API (NovaPoshtaAPI), based on the data received, I want to make a calculator with drop-down lists - a choice of cities.

A bit about the API
1) The API uses POST requests from the client
2) Cities are unloaded by the entire directory, there is no way to pull out cities one at a time.

Approach 1 (via AJAX request on client without server)

let cityObj = {
            "modelName": "Address",
            "calledMethod": "getCities",
            "methodProperties": {},
            "apiKey": apiKey
        };

        let xhr = new XMLHttpRequest();

        xhr.open('POST', 'https://api.novaposhta.ua/v2.0/json/');

        xhr.setRequestHeader('Content-type', 'application/json; charset = utf-8');

        //Отправка данных на сервер
        xhr.send(JSON.stringify(cityObj));

        //проверяем состояние запроса
        xhr.addEventListener('readystatechange', function () {
            if (xhr.readyState < 4) {

            } else if (xhr.readyState === 4 && xhr.status === 200) {
                let city = [];
                let obj = {};

                let cities = JSON.parse(xhr.response);

                for (let i = 0; i < cities.data.length; i++) {


                    obj = {
                        Ref: cities.data[i].Ref,
                        Description: cities.data[i].Description
                    };

                    //Сформировали справочник населенных пунктов
                    city.push(obj);

                }
                console.log(city); //получаем массив из 4000+ населенных пунктов

                //Далее вся логика(живой поиск, формирование выпадающего списка, другие действия)

            } else {
                console.log('Что то пошло не так');
            }
        });

The result of the work is an array of cities (there are more than 4000 records).
Next, organize a live search based on this array.
All further logic (consists of three large functions) will be described inside this request.
When the page is reloaded, the entire array of cities will be loaded again first, and then the logic.
How to avoid it?
How I think to solve:
- when the user first visits the site, the presence of cookies is checked (for example, test),
if false - an AJAX request is performed, the data is entered into localStorage and then a live search is performed from the storage. (we loaded the data once and we are working)
- based on the data in localStorage, we form drop-down lists (Sending city - Receiving city)
-------------------------------------------------- --
Option 2 (using the server part on express js)
Using the request module, make a post request to api - get a reference
And then refer to the client through your own post request via ajax

app.get('/city', function(req,res){
    if (!req.body) return res.sendStatus(400);

    let cityObj = {
        "modelName": "Address",
        "calledMethod": "getCities",
        "methodProperties": {},
        "apiKey": 'apiKey'
    };
    request.post('https://api.novaposhta.ua/v2.0/json/', {form: cityObj}, function (error, response, body) {
        let data = body; //массив с городами
        res.send(data);
    });
});

1) The problem is if I send in this way - the result of the request is "Data is invalid" - because the request {form: cityObj} is not correctly composed - how to send post requests in the request module correctly, what did I do wrong? - I took the line from the documentation

2) How to transfer data from this get request to another one? - if you always perform this request when searching - constant access to api - unloading 4000 records.

3) How can I make such a request be executed automatically when the server starts - only once - because now I start it as if in "manual mode" by going to the address from the app.get('/city') request?

I will be grateful for help.
PS Maybe I'm complicating everything and it can be much easier, I'm glad to hear it.

Completely ready-made option for selecting data from directories type of cargo
(there are only a few entries)
did like this

let cargoType = calcForm.elements['cargo-type'];
    const cargoTypeObj = {
        "modelName": "Common",
        "calledMethod": "getCargoTypes",
        "methodProperties": {},
        "apiKey": apiKey
    };

    sendAjax(cargoType, cargoTypeObj);

function sendAjax(elem, reqObj) {
    let xhr = new XMLHttpRequest();

    xhr.open('POST', 'https://api.novaposhta.ua/v2.0/json/');

    xhr.setRequestHeader('Content-type', 'application/json; charset = utf-8');

    //Отправка данных на сервер
    xhr.send(JSON.stringify(reqObj));

    //проверяем состояние запроса
    xhr.addEventListener('readystatechange', function () {
        if (xhr.readyState < 4) {

        } else if (xhr.readyState === 4 && xhr.status === 200) {

            let type = JSON.parse(xhr.response);

            for(let i = 0; i < type.data.length; i++) {

                createOptions(type.data[i]);

                elem.append(createOptions(type.data[i]));
            }

            //Формируем данные в декоративном селекте
            if(cargoType.previousElementSibling.hasAttribute('data-select')){
                let select = elem.previousElementSibling.querySelector('.form-select__dropdown');

                for(let i = 0; i < type.data.length; i++) {
                    createSelectItem(type.data[i]);
                    select.append(createSelectItem(type.data[i]));
                }
            }

        } else {
            console.log('Что то пошло не так');
        }

    });
}

//Создаем пункты реального селекта ++
function createOptions(data){
    //Формируем реальный селект
    let opt = document.createElement('option');
    opt.value = data.Ref;
    opt.textContent = data.Description;

    return opt;
}

//Создаем пункты декоративного селекта ++
function createSelectItem(data) {
    let selectItem = document.createElement('div');
    selectItem.classList.add('form-select__item');
    selectItem.setAttribute('data-select-item', data.Ref);
    selectItem.textContent = data.Description;
    return selectItem;
}

Answer the question

In order to leave comments, you need to log in

1 answer(s)

dmitriu256, 2020-05-04
@dmitriu256

Kovalsky , based on the described algorithm of actions, the following turned out
Option 1 (search for cities is carried out on the server with the return of the result to the client)
Server part (express js + nodejs)

//используем этот запрос для живого поиска
app.post('/city', function(req,res){

    let text = req.body.text;

   let val = text.trim().toLowerCase(); // приходит от пользователя
    console.log(val);

    let city = arr.filter(el => {
        if(val){
            return el.Description.toLowerCase().search(val) !== -1;
        }
    });
    
    res.send(city); // возвращаем массив с данными иначе пустой массив
});

app.listen(PORT, function(){
    console.log(`Прослушиваем порт по адресу ${PORT}`);
    //Формируем справочник городов компании
        let cityObj = {
            "modelName": "Address",
            "calledMethod": "getCities",
            "methodProperties": {},
            "apiKey": apiKey
        };
        request.post(
            'https://api.novaposhta.ua/v2.0/json/',
            {
                json: cityObj,
                headers: {
                    "Content-type": "application/json",
                }
            },
            function (error, response, body) {
                let data = body;

                for(let i = 0; i < data.data.length; i++) {
                    arr.push(data.data[i]);
                    console.log(arr[i]);
                }
            });
    });

Client-side processing

let city = calcForm.elements['cargo-city-to'];


if(city.previousElementSibling.hasAttribute('data-select')){
    let el = city.previousElementSibling.querySelector('[data-select-title]');
    el.setAttribute('contenteditable', true);

    el.addEventListener('focus', function(){
        this.textContent = '';

    });

    el.addEventListener('input', function(){
        //Очистка декоративных пунктов
        let selected = city.previousElementSibling.querySelectorAll('[data-select-item]');

        
        selected.forEach(el => {
            console.log(el.textContent);
            el.remove();
        });

        //очистка стандартного селекта
        for(let i = 0; i < city.options.length; i++){
            city.remove(i);
        }


        //Запрос живого поиска
        let xhr = new XMLHttpRequest();

        xhr.open('POST', 'http://localhost:3010/city');

        xhr.setRequestHeader('Content-type', 'application/json; charset = utf-8');

        //Отправка данных на сервер
        xhr.send(JSON.stringify({text: this.textContent}));
        console.log(this.textContent);

        //проверяем состояние запроса
        xhr.addEventListener('readystatechange', function () {
            if (xhr.readyState < 4) {

            } else if (xhr.readyState === 4 && xhr.status === 200) {

                let data = JSON.parse(xhr.response);

                for(let i = 0; i < data.length; i++) {
                    //Формируем список опций у реального селекта
                    createOptions(data[i]);

                    city.append(createOptions(data[i]));
                }

                //Формируем данные в декоративном селекте
                if(city.previousElementSibling.hasAttribute('data-select')){

                    let select = city.previousElementSibling.querySelector('.form-select__dropdown');

                    for(let i = 0; i < data.length; i++) {
                        createSelectItem(data[i]);
                        select.append(createSelectItem(data[i]));
                    }
                }

            } else {
                console.log('Что то пошло не так');
            }

        });

    });
}else{
    console.log('---');
}


getCity(city, `http://localhost:3010/city`);


//Изначально загружаем с сервера 5 записей из массива городов (что бы декоративный селект имел некоторый список по умолчанию)
function getCity(elem, url){
    let xhr = new XMLHttpRequest();

    xhr.open('GET', url);

    xhr.setRequestHeader('Content-type', 'application/json; charset = utf-8');

    //Отправка данных на сервер
    xhr.send();

    //проверяем состояние запроса
    xhr.addEventListener('readystatechange', function () {
        if (xhr.readyState < 4) {

        } else if (xhr.readyState === 4 && xhr.status === 200) {

            let data = JSON.parse(xhr.response);

            for(let i = 0; i < 5; i++) {

                createOptions(data[i]);

                elem.append(createOptions(data[i]));
            }

            //Формируем данные в декоративном селекте
            if(elem.previousElementSibling.hasAttribute('data-select')){

                let select = elem.previousElementSibling.querySelector('.form-select__dropdown');

                for(let i = 0; i < 5; i++) {
                    createSelectItem(data[i]);
                    select.append(createSelectItem(data[i]));
                }
            }

        } else {
            console.log('Что то пошло не так');
        }

    });
}

Option 2 (when all the logic happens on the client, we use the API directly, we store the cities in localStorage)

//Получение городов компании
    let cityTo = calcForm.elements['cargo-city-to'];
    let cityFrom = calcForm.elements['cargo-city'];

    checkKeyCity('Mycity', cityTo);
    checkKeyCity('Mycity', cityFrom);

    //Получение куки
    function getCookie(name) {
        let matches = document.cookie.match(new RegExp(
            "(?:^|; )" + name.replace(/([\.$?*|{}\(\)\[\]\\\/\+^])/g, '\\$1') + "=([^;]*)"
        ));
        return matches ? decodeURIComponent(matches[1]) : undefined;
    }

    //Загрузка городов в localStorage
    function getCities(){
        let date = new Date();
        date = new Date(date.setDate(date.getDate() + 1));


        let cityObj = {
            "modelName": "Address",
            "calledMethod": "getCities",
            "methodProperties": {},
            "apiKey": apiKey
        };

        let xhr = new XMLHttpRequest();

        xhr.open('POST', 'https://api.novaposhta.ua/v2.0/json/');

        xhr.setRequestHeader('Content-type', 'application/json; charset = utf-8');

        //Отправка данных на сервер
        xhr.send(JSON.stringify(cityObj));

        //проверяем состояние запроса
        xhr.addEventListener('readystatechange', function () {
            if (xhr.readyState < 4) {

            } else if (xhr.readyState === 4 && xhr.status === 200) {
                let city = [];
                let obj = {};

                let cities = JSON.parse(xhr.response);

                for (let i = 0; i < cities.data.length; i++) {


                    obj = {
                        Ref: cities.data[i].Ref,
                        Description: cities.data[i].Description
                    };

                    //Сформировали справочник населенных пунктов
                    city.push(obj);

                }

                localStorage.setItem('Mycity', JSON.stringify(city));

            } else {
                console.log('Что то пошло не так');
            }
        });

        document.cookie = `${decodeURI('cities')} = ${decodeURI(true)}; expires = ${date}; path = /`;
    }


    //Проверка наличие куки
    if(getCookie('cities') === undefined) {
        getCities();
    }

    //Проверка наличия ключа в localStorage
    function checkKeyCity(key, element){

        if (localStorage.getItem(key) !== null){
            let data = JSON.parse(localStorage.getItem(key));

            //Формируем данные в селекте
            for(let i = 0; i < data.length; i++) {

                createOptions(data[i]);

                element.append(createOptions(data[i]));
            }


            //Формируем данные в декоративном селекте
            if(element.previousElementSibling.hasAttribute('data-select')){
                let select = element.previousElementSibling.querySelector('.form-select__dropdown');
                select.overflowY = 'scroll';

                for(let i = 0; i < data.length; i++) {
                    createSelectItem(data[i]);
                    select.append(createSelectItem(data[i]));
                }
            }


            let el = element.previousElementSibling.querySelector('[data-select-title]');
            el.setAttribute('contenteditable', true);

            el.addEventListener('focus', function(){
                this.textContent = '';
            });


            //Живой поиск ++
            el.addEventListener('input', function(){

                element.previousElementSibling.querySelector('.form-select__dropdown').classList.remove('hidden');

                let val = this.textContent.trim().toLowerCase();

                let items = element.previousElementSibling.querySelectorAll('.form-select__item');


                if(val != ''){
                    items.forEach(function(elem) {
                        if(elem.textContent.toLowerCase().search(val) == -1) {
                            elem.classList.add('hidden');
                        }else{
                            elem.classList.remove('hidden');
                        }
                    });
                }else{
                    items.forEach(function(elem) {
                        elem.classList.remove('hidden');
                    });
                }
            });

        }
    }

Bottom line
I settled on Option 2 - the search speed is faster, because once we load the cities into the storage, in the future we constantly access it, without using server resources.
-Building the DOM tree happens once
- live search is done by simply adding the hidden class of cities that do not match the search conditions.