joplin-notes/joplin notion extension/Web Development/nodejs/snippets/reading large json objects.md

reading large json objects https://sentrytwo.com/14af84439a

Reading JSON objects

const fs = require('fs');
const JSONStream = require('JSONStream');
const es = require('event-stream');

const inputFile = './largeInput.json';

const readStream = fs.createReadStream(inputFile)
  .pipe(JSONStream.parse('reponse.stats.*'))
  .pipe(es.mapSync(data => {
    // Process each stat record here
    console.log(data);
    
    // For example, you can modify each object:
    // data.newField = `new value for ${data.id}`;
    // Then you might want to save it elsewhere or just handle it.
  }));

 

Writing JSON objects

const fs = require('fs');
const JSONStream = require('JSONStream');

const outputFile = './largeOutput.json';

const writeStream = fs.createWriteStream(outputFile);
const stringifyStream = JSONStream.stringify();

// Pipe the JSONStream output to your file
stringifyStream.pipe(writeStream);

// Assuming you have a way to generate or fetch your objects
for (let i = 0; i < 10000; i++) {
  const item = { id: i, name: `Item ${i}` };
  stringifyStream.write(item);
}

// Don't forget to end the stream
stringifyStream.end();

// Optionally, handle the finish event
writeStream.on('finish', () => {
  console.log('Finished writing to file');
});

Another one https://www.npmjs.com/package/big-json but at end it is merging  

Maybe this one will help which is used to run commands on json file https://www.npmjs.com/package/node-jq  

someone else posted JSONStream should work correctly for your use case, but if you can' t make it to work, you could use libraries like stream-json, oboe.js or even json-parser. Here is how you could use stream-json:

const { parser } = require('stream-json');
const { streamArray } = require('stream-json/streamers/StreamArray');
const fs = require('fs');

const jsonStream = fs.createReadStream('large-file.json').pipe(parser());

jsonStream.pipe(streamArray()).on('data', ({key, value}) => {
  console.log(value); // Processes each item individually as a JavaScript object
});