joplin-notes/joplin notion extension/Web Development/nodejs/file split and merge (express and multer).md

file split and merge (express and multer)

frontEnd

```html
	<!DOCTYPE html>
<html lang="en">
<head>
	<meta charset="UTF-8">
	<meta name="viewport" content="width=device-width, initial-scale=1.0">
	<title>Document</title>
</head>
<body>
	<input id="fileInput" type="file" name="" id="" multiple ><br>
	<button onclick="submit()" id="submit">Submit</button>
	<script src="script.js"></script>
</body>
</html>
```

js
```jsx
	const fileEle=document.querySelector("#fileInput")
const sliceSize=10*1024*1024 // 10MB

const promiseSetTimeOut = (time) => {
  return new Promise((resolve) => {
    setTimeout(() => {
      resolve();
    }, time);
  });
};

const submitIndividual=async(file,index,total)=>{
	const fileSize=file.size
	const fileName=file.name

	console.log('file: ',file)

	let start=0;
	let end=sliceSize

	const filePieces=[]
	let i=0;
	while(end<=fileSize)
	{
		const chunk = file.slice(start,end)
		if(end==fileSize)
		{
			filePieces.push({pos: i,chunk,isEnd: true,fileName})	
		}
		else{
			filePieces.push({pos: i,chunk,isEnd: false})	
		}
		
		start=end
		end+=sliceSize	
		i++
	}

	if(start<fileSize)
	{
		const chunk = file.slice(start,fileSize)
		filePieces.push({pos: i,chunk,isEnd: true,fileName})	
		i++
	}

	





	for(let filePiece of filePieces )
	{
		const form=new FormData()

		// form.append('data',{pos: filePiece.pos,isEnd: filePiece.isEnd} )
		form.append('pos',filePiece.pos )
		form.append('isEnd',filePiece.isEnd)
		form.append('file',filePiece.chunk)
		form.append('fileName',filePiece.fileName)
		form.append('total',total)
		form.append('fileIndex',index)

		const res=await fetch('/submit',{
			method: 'post',
			// headers: {
			// 	'Content-Type': 'multipart/form-data'
			// },
			body: form
		})
		const data=await res.json()

		await promiseSetTimeOut(2000)



	}	

	console.log('file: ',file)
	console.log('filePieces: ',filePieces)

}


const submit=async()=>{

	for(let file of fileEle.files)
	{
		await submitIndividual(file,i,fileEle.files.length)
		
	}
	
	
	
	// const blob=new Blob(filePieces)
	// const a=document.createElement("a")
	// const link=URL.createObjectURL(blob)
	// a.href=	link
	// a.download="something1"
	// a.click()


}
```
&nbsp;

Nodejs
```js
const express=require('express')
const fs=require('fs')
const FormData= require('form-data')

const app=express()
const PORT= process.env.PORT||8080

const multer=require('multer')
const upload = multer({ dest: 'uploads/' });

// Define the fields you want to handle
const fieldsToParse = [
{ name: 'file', maxCount: 1 },
  { name: 'pos', maxCount: 1 },
  { name: 'isEnd', maxCount: 1 },
  { name: 'fileName', maxCount: 1 },
	{ name: 'total', maxCount: 1 },
  { name: 'fileIndex', maxCount: 1 },

];


app.use(express.static(__dirname+'/front'))
app.use(express.json());
app.use(express.urlencoded({extended: true}) )

// 2d array. outer dimension containing files and inner dimension containing file pieces
const fileObjs=[]
let total=0



const getLast=(arr)=>{
	if(arr.length>0)
	return arr[arr.length-1]
	else{
		return null
	}
}

const mergeFile=()=>{
	// mergeFile will happen when only last filepiece is recieved so now
	// it will merge
	const filePieces=getLast(fileObjs)

	const arrBuffer=[]
	let ogname=""
	for(let filePiece of filePieces)
	{
		const pieceData=fs.readFileSync(`uploads/${filePiece.name}`)
		fs.unlinkSync(`uploads/${filePiece.name}`)
		arrBuffer.push(pieceData)
		ogname=filePiece.ogName


	}

	console.log('arrBuffer: ',arrBuffer)

	const mergedFileBuffer=Buffer.concat(arrBuffer)
	fs.writeFileSync(`uploads/${ogname}`,mergedFileBuffer)
}

app.post('/submit',upload.fields(fieldsToParse),(req,res)=>{

	try{
		
		if(total==0)
		{
			fileLinks.splice(0, fileLinks.length);
		}
		
		 const file = req.files['file'][0]; // Access the uploaded file
  const pos = req.body['pos']; // Access 'pos' field
  const isEnd = req.body['isEnd']; 
  const fileName=req.body['fileName']
  total= Number(req.body['total'])
  const fileIndex=Number(req.body['fileIndex'])

  // determining if current piece is beginning or not
  let isStart=false
	// const body=req.body
	// const file=req.file

	console.log('body: ',req.body)
	// console.log('body.data: ',.data)
	// console.log('body.pos: ',body.pos)
	console.log('file: ',file)
	// console.log('pos: ',pos)
	// console.log('isEnd: ',isEnd)
	// console.log('fileName: ',fileName)

	const filePieceObj={
		pos,
		isEnd,
		file,
		ogName: fileName,
		name: file.filename
	}



	// if array is empty or last of piece of last file 'isEnd' is true
	// last piece of last file bcz so that next time when file batch comes
	// then for next pieces of first file 'isStart': true
	if(fileObjs.length==0 || getLast(getLast(fileObjs)).isEnd==='true' )
	{
		isStart=true

	}
	else{
		isStart=false
	}

	if(isStart)
	{
		fileObjs.push([filePieceObj])
	}
	else{
		getLast(fileObjs).push(filePieceObj)
	}

	if(filePieceObj.isEnd==='true')
	{
		console.log('isEnd true ')
		mergeFile()
		
		// when all files are recieved here
		if(fileIndex===total-1)
		{
			console.log('final end reached')
			message+=" All files uploaded."
			total=0

			res.json({status: 'ok',message, isCompleted: true})
			return

		}
		
	}

	console.log('fileObjs: ',fileObjs)
	res.json({ status: 'ok',message: 'uploaded successfully'})

	}
	catch(err){
		console.log('err: ',err)
		res.json({status: 'error'})
	}


	


})

const server=app.listen(PORT,()=>{
	console.log('app is running in PORT ',PORT)
})
```
&nbsp;
Generalized version
Front end
```js
// submitIndividualGeneral(file,HOST/cors-submit/chunks?url=<encoded_url> ,resType,bodyType,method,bodyToBeSent,fileFieldName,chunksize)
// after fetch when link is ready message is filelink is expected and status and message field are optional
// bodyType is either 'form' or 'file'
// method is either 'post' or 'put'
const submitIndividualGeneral = async (
  file,
  url,
  resType = "json",
  bodyType = "form",
  method = "post",
  bodyToBeSent = {},
  fileFieldName = "",
  chunksize = sliceSize
) => {
  const fileSize = file.size;
  const fileName = file.name;

  try {
    console.log("file: ", file);
    let start = 0;
    let end = chunksize;

    const filePieces = [];
    let i = 0;
    while (end <= fileSize) {
      const chunk = file.slice(start, end);
      if (end == fileSize) {
        filePieces.push({ pos: i, chunk, isEnd: true, fileName });
      } else {
        filePieces.push({ pos: i, chunk, isEnd: false });
      }

      start = end;
      end += chunksize;
      i++;
    }

    if (start < fileSize) {
      const chunk = file.slice(start, fileSize);
      filePieces.push({ pos: i, chunk, isEnd: true, fileName });
      i++;
    }

    // let fileLinks = [];
    for (let filePiece of filePieces) {
      const form = new FormData();

      // form.append('data',{pos: filePiece.pos,isEnd: filePiece.isEnd} )

      let res;

      form.append("pos", filePiece.pos);
      form.append("isEnd", filePiece.isEnd);
      form.append("file", filePiece.chunk);

      form.append("fileName", filePiece.fileName);

      form.append("bodyToBeSent", JSON.stringify(bodyToBeSent));

      // body to be sent to proxy will be form but from proxy to target will be file if bodytype is file
      form.append("bodyType", bodyType);

      if (bodyType === "form") {
        form.append("fileFieldName", fileFieldName);
      } else {
        form.append("fileFieldName", "fname");
      }

      // form.append("total", total);
      // form.append("fileIndex", index);

      res = await fetch(url, {
        method,
        // headers: {
        // 	'Content-Type': 'multipart/form-data'
        // },
        body: form,
      });

      let data;
      if (resType === "json") {
        data = await res.json();
      } else if (resType === "text") {
        data = await res.text();
      }

      console.log("res: ", data);
      if (filePiece.isEnd === true) {
        // const fileLink = data;
        return { status: "ok", data };
      }

      await promiseSetTimeOut(2000);
    }

    console.log("file: ", file);
    console.log("filePieces: ", filePieces);

    return { status: "error" };
  } catch (err) {
    console.log("err: ", err);
    // return { status: "error", message: err };
    throw err;
  }
};


 const encodedURL = encodeURIComponent(apiUrl);
  // const proxyURL='http://localhost:8080/cors-submit'
  // const proxyURL = `${CORS_PROXY}/cors-submit`;
  const proxyURL = `${CORS_PROXY}/cors-submit/chunks`;

const fetchurl = `${proxyURL}?url=${encodedURL}`;

const resData = await submitIndividualGeneral(
	file,
	fetchurl,
	"text",
	"file",
	"put"
);

```

&nbsp;

### splitting in nodejs instead of browser

using buffers
```js
const fs = require('fs');

const fileBuffer = fs.readFileSync('small-file.zip'); // Entire file loaded into RAM
const CHUNK_SIZE = 50 * 1024 * 1024;

for (let i = 0; i < fileBuffer.length; i += CHUNK_SIZE) {
    const chunk = fileBuffer.slice(i, i + CHUNK_SIZE);
    fs.writeFileSync(`part-${i / CHUNK_SIZE}.part`, chunk);
}

```

using streams for efficiency
```js
const fs = require('fs');
const path = require('path');

// Configuration: 50MB in bytes
const CHUNK_SIZE = 50 * 1024 * 1024; 
const sourceFile = 'your-large-file.zip';

function splitFile(filePath) {
    const stats = fs.statSync(filePath);
    const totalParts = Math.ceil(stats.size / CHUNK_SIZE);
    const fileName = path.basename(filePath);

    for (let i = 0; i < totalParts; i++) {
        const start = i * CHUNK_SIZE;
        // The 'end' is inclusive, so we subtract 1
        const end = Math.min((i + 1) * CHUNK_SIZE - 1, stats.size - 1);
        
        const readStream = fs.createReadStream(filePath, { start, end });
        const partName = `${fileName}.part${(i + 1).toString().padStart(3, '0')}`;
        const writeStream = fs.createWriteStream(partName);

        readStream.pipe(writeStream);

        readStream.on('end', () => {
            console.log(`Finished writing ${partName}`);
        });
    }
}

splitFile(sourceFile);

```