Answer the question
In order to leave comments, you need to log in
Why is the same ID assigned to the elements?
xml-stream parses the file and gives it to the objects. These objects must be written to the database. To make it faster, I cache every 100 objects and write them to the database when 100 of them are collected. In fact, the same IDs are assigned to the objects, even if I use nanoid to generate the ID, and not take them from the xml file (where they are also unique). Why is this happening and how to fix it?
const fs = require("fs");
const XmlStream = require("xml-stream");
const MongoClient = require("mongodb").MongoClient;
(async function() {
const client = new MongoClient("mongodb://localhost:27017");
try {
await client.connect();
console.log("Connected correctly to server");
const db = client.db("opendata");
const stream = fs.createReadStream("./data/edrpou.xml");
const xml = new XmlStream(stream);
let countAll = 0;
let countNonObjNonEdrpou = 0;
let tmpBatch = [];
xml.on("endElement: SUBJECT", async function(data) {
try {
if (data instanceof Object && data.EDRPOU) {
delete data["$"];
data._id = data.EDRPOU;
if (tmpBatch.length > 100) {
tmpBatch.push(data);
await db.collection("organization").insertMany(tmpBatch);
tmpBatch = [];
} else {
tmpBatch.push(data);
}
} else {
countNonObjNonEdrpou++;
}
countAll++;
console.clear();
console.log(" ", countAll);
console.error("!object & !edrpou objects: ", countNonObjNonEdrpou);
} catch (error) {
throw new Error(error);
}
});
xml.on("end", function() {
console.error("!object & !edrpou objects: ", countNonObjNonEdrpou);
console.log(" END");
// Close connection
client.close();
});
} catch (err) {
console.log(err.stack);
}
})();
Answer the question
In order to leave comments, you need to log in
Didn't find what you were looking for?
Ask your questionAsk a Question
731 491 924 answers to any question