fedallbest/joplin-notes
Watch
1
0
Fork
You've already forked joplin-notes
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

initial

Browse source
This commit is contained in:
nm1 2026-11-04 00:33:23 +05:30
commit 74ae93a96e
4 changed files with 94691 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

204
joplin notion extension/Web Development/nodejs/snippets/json to sqlite db.md Normal file
View file

@ -0,0 +1,204 @@
json to sqlite db
```js
const jsonToSQL = (
json = "",
rootPath = ".",
dbfilename = "db.db",
tablename = "tablename",
primaryKeyField,
isAppendMode = false
) => {
const originalSchemaPath = `${rootPath}/originalSchema.json`;
const parsedSchemaPath = `${rootPath}/parsedSchema.json`;
const jsonData = JSON.parse(json);
const db = new Database(`${rootPath}/${dbfilename}`);
// console.log(typeof [1, 2]);
// console.log(jsonData);
// db.exec(`drop table if exists ${tablename}`);
// db.exec(`delete from ${tablename}`);
const isArray = Array.isArray(jsonData);
let finalData = jsonData;
// if not an array then converting to array
if (!isArray) {
finalData = Object.values(finalData);
}
let originalSchemaWrapper = {};
let parsedSchemaWrapper = {};
const originalSchema = {};
const parsedSchema = {};
// retrieving schema stored earlier
if (fs.existsSync(originalSchemaPath)) {
const value = JSON.parse(fs.readFileSync(originalSchemaPath));
originalSchemaWrapper = value;
Object.assign(originalSchema, value[tablename] ?? {});
}
if (fs.existsSync(parsedSchemaPath)) {
const value = JSON.parse(fs.readFileSync(parsedSchemaPath));
parsedSchemaWrapper = value;
Object.assign(parsedSchema, value[tablename] ?? {});
}
const createTable = () => {
if (finalData.length == 0) {
return;
}
const firstData = finalData[0];
if (typeof firstData != "object" || firstData === null) {
throw new Error("invalid format");
}
// key value pair of fieldname and type, both are strings
for (let key in firstData) {
originalSchema[key] = typeof firstData[key];
}
const sqlColumnsInfoArray = [];
for (let colname in originalSchema) {
// to check if current column name is of primary key or not
const isPrimaryKey =
primaryKeyField != undefined &&
primaryKeyField != null &&
primaryKeyField == colname;
// console.log(colname, " isPrimaryKey: ", isPrimaryKey);
if (originalSchema[colname] == "number") {
parsedSchema[colname] = "numeric";
} else if (originalSchema[colname] == "boolean") {
parsedSchema[colname] = "integer";
// everything else is text as text can even store numbers too after being parsed to string so useful for nulls
} else {
parsedSchema[colname] = "text";
}
sqlColumnsInfoArray.push(
`${colname} ${parsedSchema[colname]} ${
isPrimaryKey ? "primary key" : ""
}`
);
}
const sqlCreateQuery = `create table if not exists ${tablename} (
${sqlColumnsInfoArray.join(",")}
);`;
// console.log(sqlCreateQuery);
const result = db.exec(sqlCreateQuery);
};
const insertTable = (record) => {
if (typeof record != "object" || record == null) {
throw new Error("invalid format");
}
for (let key in record) {
if (!Object.hasOwn(originalSchema, key)) {
let newSqlColumnInfo = "";
originalSchema[key] = typeof record[key];
if (originalSchema[key] == "number") {
parsedSchema[key] = "number";
} else if (originalSchema[key] == "boolean") {
parsedSchema[key] = "integer";
// newSqlColumnsInfoArray.push(`${key} integer`);
// everything else is text as text can even store numbers too after being parsed to string so useful for nulls
} else {
parsedSchema[key] = "text";
// newSqlColumnsInfoArray.push(`${key} text `);
}
newSqlColumnInfo = `${key} ${parsedSchema[key]} `;
db.exec(`alter table ${tablename} add column ${newSqlColumnInfo}`);
}
}
const columns_string = Object.keys(record).join(",");
const valuesArray = [];
for (let column in record) {
// if text
if (parsedSchema[column] === "text") {
// check if value is null or not
if (record[column] === null) {
valuesArray.push(null);
}
// if value is string then simply pushing it
else if (typeof record[column] === "string") {
valuesArray.push(record[column]);
}
// if its number or array or object it will be stringified accordingly
else {
valuesArray.push(JSON.stringify(record[column]));
}
}
// if its boolean
else if (parsedSchema[column] === "integer") {
valuesArray.push(record[column] ? 1 : 0);
// if its number
} else {
valuesArray.push(record[column]);
}
}
// const valuesPlaceholderArray =
const values_placeholders_string = new Array(valuesArray.length)
.fill("?")
.join(",");
const insertStatement = db.prepare(
`insert into ${tablename} (${columns_string}) values (${values_placeholders_string}) `
);
const info = insertStatement.run(...valuesArray);
// console.log("info: ", info);
};
if (Object.keys(originalSchema).length === 0) {
createTable();
} else {
if (!isAppendMode) {
db.exec(`delete from ${tablename}`);
}
}
for (let record of finalData) {
insertTable(record);
}
// console.log("result: ", result);
// const insertStatement = db.exec(
// "create table if not exists channels (info text)"
// );
// console.log("insertStatement: ", insertStatement);
originalSchemaWrapper[tablename] = originalSchema;
parsedSchemaWrapper[tablename] = parsedSchema;
fs.writeFileSync(originalSchemaPath, JSON.stringify(originalSchemaWrapper));
fs.writeFileSync(parsedSchemaPath, JSON.stringify(parsedSchemaWrapper));
// console.log("done");
console.log(`converted json to table ${tablename} in db ${dbfilename}`);
};
```

267
joplin notion extension/Web Development/nodejs/snippets/padloc To Keepaas.md Normal file
View file

@ -0,0 +1,267 @@
padloc To Keepaas
`app.js`
```js
const fs =require('fs')
const {getRecords} = require('./helpers')
const filename= 'My_Vault_2024-05-13.csv'
// const filename= 'test.csv'
const outputDataFile = "output.xml"
let outputData=""
outputData+=`<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<KeePassFile>
<Meta/>
<Root>
<Group>
<Name>Root</Name>
<Notes/>
<IconID>48</IconID>
<Times/>
<IsExpanded>True</IsExpanded>
<DefaultAutoTypeSequence/>
<EnableAutoType>null</EnableAutoType>
<EnableSearching>null</EnableSearching>
<Group>
<Name>padloc</Name>
<Notes>containing prev padloc database</Notes>
<IconID>48</IconID>
<Times/>
<IsExpanded>True</IsExpanded>
<DefaultAutoTypeSequence/>
<EnableAutoType>null</EnableAutoType>
<EnableSearching>null</EnableSearching>
`
const records = getRecords(filename)
// function to map names of keys to match that of keepaas ones
const mapObjects=(obj)=>{
const newObj={}
for(let key of Object.keys(obj)){
if(obj[key].length===0){
continue;
}
// console.log('key to be mapped: ',key, ' value: ', obj[key])
switch(key){
case 'name': newObj['Title']=obj['name']; break;
case 'tags' : newObj['Tags']= obj['tags']; break;
case 'Username' : newObj['UserName'] = obj['Username']; break;
case 'Email Address' : newObj['Email'] = obj[key]; break;
case 'Plain Text' : newObj['Notes'] = obj[key]; break;
case 'Name' : newObj['UserName'] = obj[key]; break;
default : newObj[key] = obj[key]
}
}
// console.log('newObj: ',newObj)
return newObj;
}
// function to add escape sequqnce for reserved characters in xml
const mapValueForXML=(str)=>{
let newStr= str.replace(/"/g,'')
newStr = newStr.replace(/&/g,'&amp;')
newStr = newStr.replace(/</g,'&lt;')
newStr = newStr.replace(/>/g,'&gt;')
newStr = newStr.replace(/"/g,'&quot;')
newStr = newStr.replace(/'/g,'&apos;')
return newStr
}
for(let record of records){
const tags= record.tags.replace(/"/g,'')
let tagsXML=''
if(tags.length===0){
tagsXML='<Tags/>'
}
else{
tagsXML=`<Tags>${tags}</Tags>`
}
outputData+=` <Entry>
<IconID>0</IconID>
<ForegroundColor/>
<BackgroundColor/>
<OverrideURL/>
${tagsXML}
<Times/>
`
const mappedRecord = mapObjects(record)
// console.log('')
for(let key in mappedRecord){
// console.log('')
if(key==='Password'){
outputData+=` <String>
<Key>${key}</Key>
<Value ProtectInMemory="True">${ mapValueForXML(mappedRecord[key]) }</Value>
</String>
`
continue
}
outputData+=` <String>
<Key>${key}</Key>
<Value>${ mapValueForXML(mappedRecord[key]) }</Value>
</String>
`
}
outputData+=` <AutoType>
<Enabled>True</Enabled>
<DataTransferObfuscation>0</DataTransferObfuscation>
<DefaultSequence/>
</AutoType>
<History/>
</Entry>
`
}
outputData+=` </Group>
</Group>
</Root>
</KeePassFile>`
fs.writeFileSync(outputDataFile,outputData)
```
&nbsp;
`helpers.js`
```js
const fs =require('fs')
const getRecords=(filename)=>{
const data =fs.readFileSync(filename,{encoding : 'utf8'})
// normal, quotes
let state="normal"
const lines= data.split('\n')
// console.log('lines: ',lines)
// extracting headings and also removing last character ie \r from the line
let headings= lines[0].slice(0,lines[0].length-1).split(',')
console.log('headings: ',headings)
let valuei=0
let str=``
const parseLine=(line,obj)=>{
// parsing character by character
for(let i=0;i<line.length;i++){
// comma found in normal state
if(state==="normal"&& line[i]===','){
obj[headings[valuei]]=str
// resetting buffer for next value
str=``
// moving to next field index
valuei++
}
else if(state==="normal" && line[i]==='"'){
state="quotes"
str+=line[i]
}
else if(state==="quotes" && line[i]==='"' ){
state="normal"
str+=line[i]
}
else{
str+=line[i]
}
}
}
const records=[]
let obj={}
for(let i=1;i < lines.length;i++){
// const line=lines[i].slice(0,lines[i].length-1)
// removing `\r` from every record
const line=lines[i].replace('\r','')
parseLine(line,obj)
// if at end of line, state is normal then it means current record
// ends so store last field in object and reset values, pushing record in array
if(state==='normal'){
obj[headings[valuei]]=str
records.push(obj)
// resetting buffer for next value
str=``
// moving to next field index
valuei=0
obj={}
}
// if it in quotes then add newline character to current field
else{
str+='\n'
}
// const commaSeparateArray= lines[i].split(',')
// obj['name']= commaSeparateArray[0]
// obj['tags']= commaSeparateArray[1]
// obj['Username']=commaSeparateArray[2]
// obj['Password']=commaSeparateArray[3]
// obj['URL']=commaSeparateArray[4]
// obj['Email Address']=commaSeparateArray[]
// const nameRegex= /^(.*),.*/
// const tagsRegex = /^.*,("?.*"?),.*/
}
console.log('records: ',records)
return records
}
module.exports= {getRecords}
```

77
joplin notion extension/Web Development/nodejs/snippets/reading large json objects.md Normal file
View file

@ -0,0 +1,77 @@
reading large json objects
https://sentrytwo.com/14af84439a
Reading JSON objects
```js
const fs = require('fs');
const JSONStream = require('JSONStream');
const es = require('event-stream');
const inputFile = './largeInput.json';
const readStream = fs.createReadStream(inputFile)
.pipe(JSONStream.parse('reponse.stats.*'))
.pipe(es.mapSync(data => {
// Process each stat record here
console.log(data);
// For example, you can modify each object:
// data.newField = `new value for ${data.id}`;
// Then you might want to save it elsewhere or just handle it.
}));
```
&nbsp;
Writing JSON objects
```js
const fs = require('fs');
const JSONStream = require('JSONStream');
const outputFile = './largeOutput.json';
const writeStream = fs.createWriteStream(outputFile);
const stringifyStream = JSONStream.stringify();
// Pipe the JSONStream output to your file
stringifyStream.pipe(writeStream);
// Assuming you have a way to generate or fetch your objects
for (let i = 0; i < 10000; i++) {
const item = { id: i, name: `Item ${i}` };
stringifyStream.write(item);
}
// Don't forget to end the stream
stringifyStream.end();
// Optionally, handle the finish event
writeStream.on('finish', () => {
console.log('Finished writing to file');
});
```
Another one
https://www.npmjs.com/package/big-json
but at end it is merging
&nbsp;
Maybe this one will help which is used to run commands on json file
https://www.npmjs.com/package/node-jq
&nbsp;
**someone else posted**
JSONStream should work correctly for your use case, but if you can' t make it to work, you could use libraries like stream-json, oboe.js or even json-parser. Here is how you could use stream-json:
```js
const { parser } = require('stream-json');
const { streamArray } = require('stream-json/streamers/StreamArray');
const fs = require('fs');
const jsonStream = fs.createReadStream('large-file.json').pipe(parser());
jsonStream.pipe(streamArray()).on('data', ({key, value}) => {
console.log(value); // Processes each item individually as a JavaScript object
});
```

40
joplin notion extension/Web Development/nodejs/snippets/using proxy with fetch.md Normal file
View file

@ -0,0 +1,40 @@
using proxy with fetch
https://www.zenrows.com/blog/node-fetch-proxy#how-to-use-proxy-with-node-fetch
for username and password
https://github.com/TooTallNate/proxy-agents/issues/12
```js
const fetch = require('node-fetch');
const proxyUrl = 'http://proxyserver:port';
const proxyOptions = {
proxy: proxyUrl,
headers: {
'Proxy-Authorization': 'Basic ' + Buffer.from('username:password').toString('base64')
}
};
fetch('http://url-to-fetch', proxyOptions)
.then(response => response.text())
.then(data => {
console.log(data);
})
.catch(error => {
console.error('Error:', error);
});
```
Same with curl
`curl -x http://proxyserver:port --proxy-user username:password http://url-to-fetch`
for socks5
`curl -v --socks5 127.0.0.1:1080 https://icanhazip.com`
replace with socks4 for socks4