O
O
Oleg Vdovenko2020-02-01 13:49:16
JavaScript
Oleg Vdovenko, 2020-02-01 13:49:16

Why is the same ID assigned to the elements?

xml-stream parses the file and gives it to the objects. These objects must be written to the database. To make it faster, I cache every 100 objects and write them to the database when 100 of them are collected. In fact, the same IDs are assigned to the objects, even if I use nanoid to generate the ID, and not take them from the xml file (where they are also unique). Why is this happening and how to fix it?

const fs = require("fs");
const XmlStream = require("xml-stream");
const MongoClient = require("mongodb").MongoClient;

(async function() {
  const client = new MongoClient("mongodb://localhost:27017");

  try {
    await client.connect();
    console.log("Connected correctly to server");
    const db = client.db("opendata");

    const stream = fs.createReadStream("./data/edrpou.xml");
    const xml = new XmlStream(stream);
    let countAll = 0;
    let countNonObjNonEdrpou = 0;

    let tmpBatch = [];

    xml.on("endElement: SUBJECT", async function(data) {
      try {
        if (data instanceof Object && data.EDRPOU) {
          delete data["$"];
          data._id = data.EDRPOU;
          if (tmpBatch.length > 100) {
            tmpBatch.push(data);
            await db.collection("organization").insertMany(tmpBatch);
            tmpBatch = [];
          } else {
            tmpBatch.push(data);
          }
        } else {
          countNonObjNonEdrpou++;
        }

        countAll++;
        console.clear();
        console.log("  ", countAll);
        console.error("!object & !edrpou objects:  ", countNonObjNonEdrpou);
      } catch (error) {
        throw new Error(error);
      }
    });

    xml.on("end", function() {
      console.error("!object & !edrpou objects:  ", countNonObjNonEdrpou);
      console.log("  END");
      // Close connection
      client.close();
    });
  } catch (err) {
    console.log(err.stack);
  }
})();

Answer the question

In order to leave comments, you need to log in

Didn't find what you were looking for?

Ask your question

Ask a Question

731 491 924 answers to any question