9.8 KiB
9.8 KiB
file split and merge (express and multer)
frontEnd
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Document</title>
</head>
<body>
<input id="fileInput" type="file" name="" id="" multiple ><br>
<button onclick="submit()" id="submit">Submit</button>
<script src="script.js"></script>
</body>
</html>
js
const fileEle=document.querySelector("#fileInput")
const sliceSize=10*1024*1024 // 10MB
const promiseSetTimeOut = (time) => {
return new Promise((resolve) => {
setTimeout(() => {
resolve();
}, time);
});
};
const submitIndividual=async(file,index,total)=>{
const fileSize=file.size
const fileName=file.name
console.log('file: ',file)
let start=0;
let end=sliceSize
const filePieces=[]
let i=0;
while(end<=fileSize)
{
const chunk = file.slice(start,end)
if(end==fileSize)
{
filePieces.push({pos: i,chunk,isEnd: true,fileName})
}
else{
filePieces.push({pos: i,chunk,isEnd: false})
}
start=end
end+=sliceSize
i++
}
if(start<fileSize)
{
const chunk = file.slice(start,fileSize)
filePieces.push({pos: i,chunk,isEnd: true,fileName})
i++
}
for(let filePiece of filePieces )
{
const form=new FormData()
// form.append('data',{pos: filePiece.pos,isEnd: filePiece.isEnd} )
form.append('pos',filePiece.pos )
form.append('isEnd',filePiece.isEnd)
form.append('file',filePiece.chunk)
form.append('fileName',filePiece.fileName)
form.append('total',total)
form.append('fileIndex',index)
const res=await fetch('/submit',{
method: 'post',
// headers: {
// 'Content-Type': 'multipart/form-data'
// },
body: form
})
const data=await res.json()
await promiseSetTimeOut(2000)
}
console.log('file: ',file)
console.log('filePieces: ',filePieces)
}
const submit=async()=>{
for(let file of fileEle.files)
{
await submitIndividual(file,i,fileEle.files.length)
}
// const blob=new Blob(filePieces)
// const a=document.createElement("a")
// const link=URL.createObjectURL(blob)
// a.href= link
// a.download="something1"
// a.click()
}
Nodejs
const express=require('express')
const fs=require('fs')
const FormData= require('form-data')
const app=express()
const PORT= process.env.PORT||8080
const multer=require('multer')
const upload = multer({ dest: 'uploads/' });
// Define the fields you want to handle
const fieldsToParse = [
{ name: 'file', maxCount: 1 },
{ name: 'pos', maxCount: 1 },
{ name: 'isEnd', maxCount: 1 },
{ name: 'fileName', maxCount: 1 },
{ name: 'total', maxCount: 1 },
{ name: 'fileIndex', maxCount: 1 },
];
app.use(express.static(__dirname+'/front'))
app.use(express.json());
app.use(express.urlencoded({extended: true}) )
// 2d array. outer dimension containing files and inner dimension containing file pieces
const fileObjs=[]
let total=0
const getLast=(arr)=>{
if(arr.length>0)
return arr[arr.length-1]
else{
return null
}
}
const mergeFile=()=>{
// mergeFile will happen when only last filepiece is recieved so now
// it will merge
const filePieces=getLast(fileObjs)
const arrBuffer=[]
let ogname=""
for(let filePiece of filePieces)
{
const pieceData=fs.readFileSync(`uploads/${filePiece.name}`)
fs.unlinkSync(`uploads/${filePiece.name}`)
arrBuffer.push(pieceData)
ogname=filePiece.ogName
}
console.log('arrBuffer: ',arrBuffer)
const mergedFileBuffer=Buffer.concat(arrBuffer)
fs.writeFileSync(`uploads/${ogname}`,mergedFileBuffer)
}
app.post('/submit',upload.fields(fieldsToParse),(req,res)=>{
try{
if(total==0)
{
fileLinks.splice(0, fileLinks.length);
}
const file = req.files['file'][0]; // Access the uploaded file
const pos = req.body['pos']; // Access 'pos' field
const isEnd = req.body['isEnd'];
const fileName=req.body['fileName']
total= Number(req.body['total'])
const fileIndex=Number(req.body['fileIndex'])
// determining if current piece is beginning or not
let isStart=false
// const body=req.body
// const file=req.file
console.log('body: ',req.body)
// console.log('body.data: ',.data)
// console.log('body.pos: ',body.pos)
console.log('file: ',file)
// console.log('pos: ',pos)
// console.log('isEnd: ',isEnd)
// console.log('fileName: ',fileName)
const filePieceObj={
pos,
isEnd,
file,
ogName: fileName,
name: file.filename
}
// if array is empty or last of piece of last file 'isEnd' is true
// last piece of last file bcz so that next time when file batch comes
// then for next pieces of first file 'isStart': true
if(fileObjs.length==0 || getLast(getLast(fileObjs)).isEnd==='true' )
{
isStart=true
}
else{
isStart=false
}
if(isStart)
{
fileObjs.push([filePieceObj])
}
else{
getLast(fileObjs).push(filePieceObj)
}
if(filePieceObj.isEnd==='true')
{
console.log('isEnd true ')
mergeFile()
// when all files are recieved here
if(fileIndex===total-1)
{
console.log('final end reached')
message+=" All files uploaded."
total=0
res.json({status: 'ok',message, isCompleted: true})
return
}
}
console.log('fileObjs: ',fileObjs)
res.json({ status: 'ok',message: 'uploaded successfully'})
}
catch(err){
console.log('err: ',err)
res.json({status: 'error'})
}
})
const server=app.listen(PORT,()=>{
console.log('app is running in PORT ',PORT)
})
Generalized version Front end
// submitIndividualGeneral(file,HOST/cors-submit/chunks?url=<encoded_url> ,resType,bodyType,method,bodyToBeSent,fileFieldName,chunksize)
// after fetch when link is ready message is filelink is expected and status and message field are optional
// bodyType is either 'form' or 'file'
// method is either 'post' or 'put'
const submitIndividualGeneral = async (
file,
url,
resType = "json",
bodyType = "form",
method = "post",
bodyToBeSent = {},
fileFieldName = "",
chunksize = sliceSize
) => {
const fileSize = file.size;
const fileName = file.name;
try {
console.log("file: ", file);
let start = 0;
let end = chunksize;
const filePieces = [];
let i = 0;
while (end <= fileSize) {
const chunk = file.slice(start, end);
if (end == fileSize) {
filePieces.push({ pos: i, chunk, isEnd: true, fileName });
} else {
filePieces.push({ pos: i, chunk, isEnd: false });
}
start = end;
end += chunksize;
i++;
}
if (start < fileSize) {
const chunk = file.slice(start, fileSize);
filePieces.push({ pos: i, chunk, isEnd: true, fileName });
i++;
}
// let fileLinks = [];
for (let filePiece of filePieces) {
const form = new FormData();
// form.append('data',{pos: filePiece.pos,isEnd: filePiece.isEnd} )
let res;
form.append("pos", filePiece.pos);
form.append("isEnd", filePiece.isEnd);
form.append("file", filePiece.chunk);
form.append("fileName", filePiece.fileName);
form.append("bodyToBeSent", JSON.stringify(bodyToBeSent));
// body to be sent to proxy will be form but from proxy to target will be file if bodytype is file
form.append("bodyType", bodyType);
if (bodyType === "form") {
form.append("fileFieldName", fileFieldName);
} else {
form.append("fileFieldName", "fname");
}
// form.append("total", total);
// form.append("fileIndex", index);
res = await fetch(url, {
method,
// headers: {
// 'Content-Type': 'multipart/form-data'
// },
body: form,
});
let data;
if (resType === "json") {
data = await res.json();
} else if (resType === "text") {
data = await res.text();
}
console.log("res: ", data);
if (filePiece.isEnd === true) {
// const fileLink = data;
return { status: "ok", data };
}
await promiseSetTimeOut(2000);
}
console.log("file: ", file);
console.log("filePieces: ", filePieces);
return { status: "error" };
} catch (err) {
console.log("err: ", err);
// return { status: "error", message: err };
throw err;
}
};
const encodedURL = encodeURIComponent(apiUrl);
// const proxyURL='http://localhost:8080/cors-submit'
// const proxyURL = `${CORS_PROXY}/cors-submit`;
const proxyURL = `${CORS_PROXY}/cors-submit/chunks`;
const fetchurl = `${proxyURL}?url=${encodedURL}`;
const resData = await submitIndividualGeneral(
file,
fetchurl,
"text",
"file",
"put"
);
splitting in nodejs instead of browser
using buffers
const fs = require('fs');
const fileBuffer = fs.readFileSync('small-file.zip'); // Entire file loaded into RAM
const CHUNK_SIZE = 50 * 1024 * 1024;
for (let i = 0; i < fileBuffer.length; i += CHUNK_SIZE) {
const chunk = fileBuffer.slice(i, i + CHUNK_SIZE);
fs.writeFileSync(`part-${i / CHUNK_SIZE}.part`, chunk);
}
using streams for efficiency
const fs = require('fs');
const path = require('path');
// Configuration: 50MB in bytes
const CHUNK_SIZE = 50 * 1024 * 1024;
const sourceFile = 'your-large-file.zip';
function splitFile(filePath) {
const stats = fs.statSync(filePath);
const totalParts = Math.ceil(stats.size / CHUNK_SIZE);
const fileName = path.basename(filePath);
for (let i = 0; i < totalParts; i++) {
const start = i * CHUNK_SIZE;
// The 'end' is inclusive, so we subtract 1
const end = Math.min((i + 1) * CHUNK_SIZE - 1, stats.size - 1);
const readStream = fs.createReadStream(filePath, { start, end });
const partName = `${fileName}.part${(i + 1).toString().padStart(3, '0')}`;
const writeStream = fs.createWriteStream(partName);
readStream.pipe(writeStream);
readStream.on('end', () => {
console.log(`Finished writing ${partName}`);
});
}
}
splitFile(sourceFile);