initial

joplin notion extension/Web Development/nodejs/snippets/reading large json objects.md

@@ -0,0 +1,77 @@
+reading large json objects
+https://sentrytwo.com/14af84439a
+
+Reading JSON objects
+```js
+const fs = require('fs');
+const JSONStream = require('JSONStream');
+const es = require('event-stream');
+
+const inputFile = './largeInput.json';
+
+const readStream = fs.createReadStream(inputFile)
+  .pipe(JSONStream.parse('reponse.stats.*'))
+  .pipe(es.mapSync(data => {
+    // Process each stat record here
+    console.log(data);
+    
+    // For example, you can modify each object:
+    // data.newField = `new value for ${data.id}`;
+    // Then you might want to save it elsewhere or just handle it.
+  }));
+```
+ 
+
+Writing JSON objects
+```js
+const fs = require('fs');
+const JSONStream = require('JSONStream');
+
+const outputFile = './largeOutput.json';
+
+const writeStream = fs.createWriteStream(outputFile);
+const stringifyStream = JSONStream.stringify();
+
+// Pipe the JSONStream output to your file
+stringifyStream.pipe(writeStream);
+
+// Assuming you have a way to generate or fetch your objects
+for (let i = 0; i < 10000; i++) {
+  const item = { id: i, name: `Item ${i}` };
+  stringifyStream.write(item);
+}
+
+// Don't forget to end the stream
+stringifyStream.end();
+
+// Optionally, handle the finish event
+writeStream.on('finish', () => {
+  console.log('Finished writing to file');
+});
+
+```
+
+
+Another one
+https://www.npmjs.com/package/big-json
+but at end it is merging
+&nbsp;
+
+Maybe this one will help which is used to run commands on json file
+https://www.npmjs.com/package/node-jq
+&nbsp;
+
+**someone else posted**
+JSONStream should work correctly for your use case, but if you can' t make it to work, you could use libraries like stream-json, oboe.js or even json-parser. Here is how you could use stream-json:
+```js
+const { parser } = require('stream-json');
+const { streamArray } = require('stream-json/streamers/StreamArray');
+const fs = require('fs');
+
+const jsonStream = fs.createReadStream('large-file.json').pipe(parser());
+
+jsonStream.pipe(streamArray()).on('data', ({key, value}) => {
+  console.log(value); // Processes each item individually as a JavaScript object
+});
+
+```