initial
This commit is contained in:
nm1
commit
74ae93a96e
1 changed files with 94691 additions and 0 deletions
|
|
@ -0,0 +1,470 @@
|
|||
file split and merge (express and multer)
|
||||
|
||||
frontEnd
|
||||
|
||||
```html
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>Document</title>
|
||||
</head>
|
||||
<body>
|
||||
<input id="fileInput" type="file" name="" id="" multiple ><br>
|
||||
<button onclick="submit()" id="submit">Submit</button>
|
||||
<script src="script.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
```
|
||||
|
||||
js
|
||||
```jsx
|
||||
const fileEle=document.querySelector("#fileInput")
|
||||
const sliceSize=10*1024*1024 // 10MB
|
||||
|
||||
const promiseSetTimeOut = (time) => {
|
||||
return new Promise((resolve) => {
|
||||
setTimeout(() => {
|
||||
resolve();
|
||||
}, time);
|
||||
});
|
||||
};
|
||||
|
||||
const submitIndividual=async(file,index,total)=>{
|
||||
const fileSize=file.size
|
||||
const fileName=file.name
|
||||
|
||||
console.log('file: ',file)
|
||||
|
||||
let start=0;
|
||||
let end=sliceSize
|
||||
|
||||
const filePieces=[]
|
||||
let i=0;
|
||||
while(end<=fileSize)
|
||||
{
|
||||
const chunk = file.slice(start,end)
|
||||
if(end==fileSize)
|
||||
{
|
||||
filePieces.push({pos: i,chunk,isEnd: true,fileName})
|
||||
}
|
||||
else{
|
||||
filePieces.push({pos: i,chunk,isEnd: false})
|
||||
}
|
||||
|
||||
start=end
|
||||
end+=sliceSize
|
||||
i++
|
||||
}
|
||||
|
||||
if(start<fileSize)
|
||||
{
|
||||
const chunk = file.slice(start,fileSize)
|
||||
filePieces.push({pos: i,chunk,isEnd: true,fileName})
|
||||
i++
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
for(let filePiece of filePieces )
|
||||
{
|
||||
const form=new FormData()
|
||||
|
||||
// form.append('data',{pos: filePiece.pos,isEnd: filePiece.isEnd} )
|
||||
form.append('pos',filePiece.pos )
|
||||
form.append('isEnd',filePiece.isEnd)
|
||||
form.append('file',filePiece.chunk)
|
||||
form.append('fileName',filePiece.fileName)
|
||||
form.append('total',total)
|
||||
form.append('fileIndex',index)
|
||||
|
||||
const res=await fetch('/submit',{
|
||||
method: 'post',
|
||||
// headers: {
|
||||
// 'Content-Type': 'multipart/form-data'
|
||||
// },
|
||||
body: form
|
||||
})
|
||||
const data=await res.json()
|
||||
|
||||
await promiseSetTimeOut(2000)
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
console.log('file: ',file)
|
||||
console.log('filePieces: ',filePieces)
|
||||
|
||||
}
|
||||
|
||||
|
||||
const submit=async()=>{
|
||||
|
||||
for(let file of fileEle.files)
|
||||
{
|
||||
await submitIndividual(file,i,fileEle.files.length)
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
// const blob=new Blob(filePieces)
|
||||
// const a=document.createElement("a")
|
||||
// const link=URL.createObjectURL(blob)
|
||||
// a.href= link
|
||||
// a.download="something1"
|
||||
// a.click()
|
||||
|
||||
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
Nodejs
|
||||
```js
|
||||
const express=require('express')
|
||||
const fs=require('fs')
|
||||
const FormData= require('form-data')
|
||||
|
||||
const app=express()
|
||||
const PORT= process.env.PORT||8080
|
||||
|
||||
const multer=require('multer')
|
||||
const upload = multer({ dest: 'uploads/' });
|
||||
|
||||
// Define the fields you want to handle
|
||||
const fieldsToParse = [
|
||||
{ name: 'file', maxCount: 1 },
|
||||
{ name: 'pos', maxCount: 1 },
|
||||
{ name: 'isEnd', maxCount: 1 },
|
||||
{ name: 'fileName', maxCount: 1 },
|
||||
{ name: 'total', maxCount: 1 },
|
||||
{ name: 'fileIndex', maxCount: 1 },
|
||||
|
||||
];
|
||||
|
||||
|
||||
app.use(express.static(__dirname+'/front'))
|
||||
app.use(express.json());
|
||||
app.use(express.urlencoded({extended: true}) )
|
||||
|
||||
// 2d array. outer dimension containing files and inner dimension containing file pieces
|
||||
const fileObjs=[]
|
||||
let total=0
|
||||
|
||||
|
||||
|
||||
const getLast=(arr)=>{
|
||||
if(arr.length>0)
|
||||
return arr[arr.length-1]
|
||||
else{
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
const mergeFile=()=>{
|
||||
// mergeFile will happen when only last filepiece is recieved so now
|
||||
// it will merge
|
||||
const filePieces=getLast(fileObjs)
|
||||
|
||||
const arrBuffer=[]
|
||||
let ogname=""
|
||||
for(let filePiece of filePieces)
|
||||
{
|
||||
const pieceData=fs.readFileSync(`uploads/${filePiece.name}`)
|
||||
fs.unlinkSync(`uploads/${filePiece.name}`)
|
||||
arrBuffer.push(pieceData)
|
||||
ogname=filePiece.ogName
|
||||
|
||||
|
||||
}
|
||||
|
||||
console.log('arrBuffer: ',arrBuffer)
|
||||
|
||||
const mergedFileBuffer=Buffer.concat(arrBuffer)
|
||||
fs.writeFileSync(`uploads/${ogname}`,mergedFileBuffer)
|
||||
}
|
||||
|
||||
app.post('/submit',upload.fields(fieldsToParse),(req,res)=>{
|
||||
|
||||
try{
|
||||
|
||||
if(total==0)
|
||||
{
|
||||
fileLinks.splice(0, fileLinks.length);
|
||||
}
|
||||
|
||||
const file = req.files['file'][0]; // Access the uploaded file
|
||||
const pos = req.body['pos']; // Access 'pos' field
|
||||
const isEnd = req.body['isEnd'];
|
||||
const fileName=req.body['fileName']
|
||||
total= Number(req.body['total'])
|
||||
const fileIndex=Number(req.body['fileIndex'])
|
||||
|
||||
// determining if current piece is beginning or not
|
||||
let isStart=false
|
||||
// const body=req.body
|
||||
// const file=req.file
|
||||
|
||||
console.log('body: ',req.body)
|
||||
// console.log('body.data: ',.data)
|
||||
// console.log('body.pos: ',body.pos)
|
||||
console.log('file: ',file)
|
||||
// console.log('pos: ',pos)
|
||||
// console.log('isEnd: ',isEnd)
|
||||
// console.log('fileName: ',fileName)
|
||||
|
||||
const filePieceObj={
|
||||
pos,
|
||||
isEnd,
|
||||
file,
|
||||
ogName: fileName,
|
||||
name: file.filename
|
||||
}
|
||||
|
||||
|
||||
|
||||
// if array is empty or last of piece of last file 'isEnd' is true
|
||||
// last piece of last file bcz so that next time when file batch comes
|
||||
// then for next pieces of first file 'isStart': true
|
||||
if(fileObjs.length==0 || getLast(getLast(fileObjs)).isEnd==='true' )
|
||||
{
|
||||
isStart=true
|
||||
|
||||
}
|
||||
else{
|
||||
isStart=false
|
||||
}
|
||||
|
||||
if(isStart)
|
||||
{
|
||||
fileObjs.push([filePieceObj])
|
||||
}
|
||||
else{
|
||||
getLast(fileObjs).push(filePieceObj)
|
||||
}
|
||||
|
||||
if(filePieceObj.isEnd==='true')
|
||||
{
|
||||
console.log('isEnd true ')
|
||||
mergeFile()
|
||||
|
||||
// when all files are recieved here
|
||||
if(fileIndex===total-1)
|
||||
{
|
||||
console.log('final end reached')
|
||||
message+=" All files uploaded."
|
||||
total=0
|
||||
|
||||
res.json({status: 'ok',message, isCompleted: true})
|
||||
return
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
console.log('fileObjs: ',fileObjs)
|
||||
res.json({ status: 'ok',message: 'uploaded successfully'})
|
||||
|
||||
}
|
||||
catch(err){
|
||||
console.log('err: ',err)
|
||||
res.json({status: 'error'})
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
})
|
||||
|
||||
const server=app.listen(PORT,()=>{
|
||||
console.log('app is running in PORT ',PORT)
|
||||
})
|
||||
```
|
||||
|
||||
Generalized version
|
||||
Front end
|
||||
```js
|
||||
// submitIndividualGeneral(file,HOST/cors-submit/chunks?url=<encoded_url> ,resType,bodyType,method,bodyToBeSent,fileFieldName,chunksize)
|
||||
// after fetch when link is ready message is filelink is expected and status and message field are optional
|
||||
// bodyType is either 'form' or 'file'
|
||||
// method is either 'post' or 'put'
|
||||
const submitIndividualGeneral = async (
|
||||
file,
|
||||
url,
|
||||
resType = "json",
|
||||
bodyType = "form",
|
||||
method = "post",
|
||||
bodyToBeSent = {},
|
||||
fileFieldName = "",
|
||||
chunksize = sliceSize
|
||||
) => {
|
||||
const fileSize = file.size;
|
||||
const fileName = file.name;
|
||||
|
||||
try {
|
||||
console.log("file: ", file);
|
||||
let start = 0;
|
||||
let end = chunksize;
|
||||
|
||||
const filePieces = [];
|
||||
let i = 0;
|
||||
while (end <= fileSize) {
|
||||
const chunk = file.slice(start, end);
|
||||
if (end == fileSize) {
|
||||
filePieces.push({ pos: i, chunk, isEnd: true, fileName });
|
||||
} else {
|
||||
filePieces.push({ pos: i, chunk, isEnd: false });
|
||||
}
|
||||
|
||||
start = end;
|
||||
end += chunksize;
|
||||
i++;
|
||||
}
|
||||
|
||||
if (start < fileSize) {
|
||||
const chunk = file.slice(start, fileSize);
|
||||
filePieces.push({ pos: i, chunk, isEnd: true, fileName });
|
||||
i++;
|
||||
}
|
||||
|
||||
// let fileLinks = [];
|
||||
for (let filePiece of filePieces) {
|
||||
const form = new FormData();
|
||||
|
||||
// form.append('data',{pos: filePiece.pos,isEnd: filePiece.isEnd} )
|
||||
|
||||
let res;
|
||||
|
||||
form.append("pos", filePiece.pos);
|
||||
form.append("isEnd", filePiece.isEnd);
|
||||
form.append("file", filePiece.chunk);
|
||||
|
||||
form.append("fileName", filePiece.fileName);
|
||||
|
||||
form.append("bodyToBeSent", JSON.stringify(bodyToBeSent));
|
||||
|
||||
// body to be sent to proxy will be form but from proxy to target will be file if bodytype is file
|
||||
form.append("bodyType", bodyType);
|
||||
|
||||
if (bodyType === "form") {
|
||||
form.append("fileFieldName", fileFieldName);
|
||||
} else {
|
||||
form.append("fileFieldName", "fname");
|
||||
}
|
||||
|
||||
// form.append("total", total);
|
||||
// form.append("fileIndex", index);
|
||||
|
||||
res = await fetch(url, {
|
||||
method,
|
||||
// headers: {
|
||||
// 'Content-Type': 'multipart/form-data'
|
||||
// },
|
||||
body: form,
|
||||
});
|
||||
|
||||
let data;
|
||||
if (resType === "json") {
|
||||
data = await res.json();
|
||||
} else if (resType === "text") {
|
||||
data = await res.text();
|
||||
}
|
||||
|
||||
console.log("res: ", data);
|
||||
if (filePiece.isEnd === true) {
|
||||
// const fileLink = data;
|
||||
return { status: "ok", data };
|
||||
}
|
||||
|
||||
await promiseSetTimeOut(2000);
|
||||
}
|
||||
|
||||
console.log("file: ", file);
|
||||
console.log("filePieces: ", filePieces);
|
||||
|
||||
return { status: "error" };
|
||||
} catch (err) {
|
||||
console.log("err: ", err);
|
||||
// return { status: "error", message: err };
|
||||
throw err;
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
const encodedURL = encodeURIComponent(apiUrl);
|
||||
// const proxyURL='http://localhost:8080/cors-submit'
|
||||
// const proxyURL = `${CORS_PROXY}/cors-submit`;
|
||||
const proxyURL = `${CORS_PROXY}/cors-submit/chunks`;
|
||||
|
||||
const fetchurl = `${proxyURL}?url=${encodedURL}`;
|
||||
|
||||
const resData = await submitIndividualGeneral(
|
||||
file,
|
||||
fetchurl,
|
||||
"text",
|
||||
"file",
|
||||
"put"
|
||||
);
|
||||
|
||||
```
|
||||
|
||||
|
||||
|
||||
### splitting in nodejs instead of browser
|
||||
|
||||
using buffers
|
||||
```js
|
||||
const fs = require('fs');
|
||||
|
||||
const fileBuffer = fs.readFileSync('small-file.zip'); // Entire file loaded into RAM
|
||||
const CHUNK_SIZE = 50 * 1024 * 1024;
|
||||
|
||||
for (let i = 0; i < fileBuffer.length; i += CHUNK_SIZE) {
|
||||
const chunk = fileBuffer.slice(i, i + CHUNK_SIZE);
|
||||
fs.writeFileSync(`part-${i / CHUNK_SIZE}.part`, chunk);
|
||||
}
|
||||
|
||||
```
|
||||
|
||||
using streams for efficiency
|
||||
```js
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
// Configuration: 50MB in bytes
|
||||
const CHUNK_SIZE = 50 * 1024 * 1024;
|
||||
const sourceFile = 'your-large-file.zip';
|
||||
|
||||
function splitFile(filePath) {
|
||||
const stats = fs.statSync(filePath);
|
||||
const totalParts = Math.ceil(stats.size / CHUNK_SIZE);
|
||||
const fileName = path.basename(filePath);
|
||||
|
||||
for (let i = 0; i < totalParts; i++) {
|
||||
const start = i * CHUNK_SIZE;
|
||||
// The 'end' is inclusive, so we subtract 1
|
||||
const end = Math.min((i + 1) * CHUNK_SIZE - 1, stats.size - 1);
|
||||
|
||||
const readStream = fs.createReadStream(filePath, { start, end });
|
||||
const partName = `${fileName}.part${(i + 1).toString().padStart(3, '0')}`;
|
||||
const writeStream = fs.createWriteStream(partName);
|
||||
|
||||
readStream.pipe(writeStream);
|
||||
|
||||
readStream.on('end', () => {
|
||||
console.log(`Finished writing ${partName}`);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
splitFile(sourceFile);
|
||||
|
||||
```
|
||||
Loading…
Reference in a new issue