I have a problem with an app inconsistently running queries and js objects that I’m having trouble identifying what the problem is, or even where to look. Pages won’t load consistently (some queries fail on load, but only sometimes) - I assume that this might be a time out error ?
The other problem is that a js object may run once ok, but next time it may hang for 5 or so minutes, or even return inconsistent results. I’ve checked the javascript console and can’t see any errors. Reloading the browser window seems to fix it (but is really not an ideal fix).
I have what I imagine is a moderately complex app for managing the marshalling of logs for export. Containing ~ 100,000 logs delivered by ~2000 trucks by a dozen different contractors and for which a sustainability audit trail needs to be kept. Lots of emphasis on verifying data integrity. Its running a self hosted mysql (8.0.29) data base as the back end (1 core, 2 threads 16GB) and appsmith 1.7.11 on a virtual machine via docker (Ubuntu 22.04, 1 core, 2 threads 16GB), both hosted in the same data centre (Oracle Cloud).
When an application page loads it makes around 10 queries to mysql - loading some full tables, but primarily keys and foreign keys from tables. Page takes up around 60 MB of RAM, I did have some problems with javascript promises not fulfilling and getting DataClone errors, so used await statements on my queries done via JS Objects.
I’ve been programming for 40 years, but hardly ever using javascript, so excuse my naive code below, would be grateful for any suggestions of how to tackle finding the problem.
export default {
parse_data: async () => {
//console.log("starting");
try {
await Select_Raw_WeighBridge.run();
await Lengths.run();
await Dockets.run();
await Supplier.run();
await Operations.run();
await Harvestor.run();
await Coupe.run();
await Grade.run();
await FPP.run();
//console.log("getting data");
const wb_data = Select_Raw_WeighBridge.data;
//console.log("wb");
const Len_set = new Set(Lengths.data.map(item => item.Lengths));
//console.log("ls");
const Docket_set = new Set(Dockets.data.map(item => item.Docket));
//console.log("ds");
const Supplier_set = new Set(Supplier.data.map(item => item.Supplier));
//console.log("ss");
const Operations_set = new Set(Operations.data.map(item => item.Operations));
//console.log("os");
const Harvestor_set = new Set(Harvestor.data.map(item => item.Harvestor));
//console.log("hs");
const Coupe_set = new Set(Coupe.data.map(item => item.Coupe));
//console.log("cs");
const Grade_set = new Set(Grade.data.map(item => item.Grade));
//console.log("gs");
const FPP_set = new Set(FPP.data.map(item => item.FPP));
const new_Harvestor = new Set();
const new_Operations = new Set();
const new_Supplier = new Set();
const new_Operations_Date = {};
const new_Coupe = new Set();
const new_Coupe_Date = {};
const new_Dockets = new Set();
const new_FPP = new Set();
const new_FPPExpiry = {};
//console.log("got data");
const WeighBridgeCreator = (WB_Docket, Docket, DateTime, Rego, Supplier, Operations, Coupe, Harvestor, Gross, Tare, GMT, Product, FPP, ExpiryDate) => {
const product = Product.replace(/\s/g, '').toUpperCase().split('-');
const docketData = {
Docket,
WB_Docket,
Date: DateTime.split(' ')[0].split('/').reverse().join("-"),
Time: DateTime.split(' ')[1] + ":00",
Supplier: Supplier.toUpperCase().replace(/\s/g, ''),
Operation: Operations.toUpperCase().replace(/\s/g, ''),
Coupe: Coupe.toUpperCase().replace(/\s/g, ''),
Rego: Rego.toUpperCase().trim().split(/[\s\\-]/)[0],
Harvestor: Harvestor.toUpperCase().trim().replace(/\/|\\/g, '').replace(/\s\s+/g, ' ').replace(/HA /g, 'HA / '),
Reason: '',
Gross: Gross / 1000,
Tare: Tare / 1000,
GMT: GMT / 1000,
FPP,
Expiry: ExpiryDate.split(' ')[0].split('/').reverse().join("-"),
}
if (product.length === 2) {
return {
...docketData,
Grade: product[0],
Length: isNaN(product[1]) ? 0 : product[1],
}
}
return {
...docketData,
Grade: Product,
Length: 0,
}
}
//console.log("starting processing");
const {
iv_wb,
wb,
} = wb_data.reduce(
(acc, sr) => {
const wbr = WeighBridgeCreator(sr.DOCKETNO, sr.MANUALDOCKETNO, sr.TRANSDATE, sr.REGISTRATIONNO, sr.CONTACTNAME, sr.INSTRUCTIONS1, sr.ORDERNO,
sr.INSTRUCTIONS2, sr.GROSS, sr.TARE, sr.NET, sr.PRODUCT, sr.STREETNUMBER, sr.STREET);
const invalid = reason => {
wbr.Reason += reason;
//console.log("Invalid :",wbr.Reason);
acc.iv_wb.push([wbr.WB_Docket, wbr.Docket, wbr.Date, wbr.Rego, wbr.Supplier, wbr.Operation, wbr.Coupe, wbr.Harvestor, wbr.FPP,
wbr.Gross, wbr.Tare, wbr.GMT, wbr.Grade, wbr.Length, wbr.Time, wbr.Reason]);
};
const valid = () => acc.wb.push([wbr.Docket, wbr.WB_Docket, wbr.Date, wbr.Rego, wbr.Supplier, wbr.Operation, wbr.Coupe, wbr.Harvestor, wbr.FPP,
wbr.Gross, wbr.Tare, wbr.GMT, wbr.Grade, wbr.Length]);
if (!Coupe_set.has(wbr.Coupe) && wbr.Coupe && (wbr.Coupe != 'TEST')) {
//console.log(wbr.Coupe,wbr.Date);
new_Coupe.add(wbr.Coupe);
new_Coupe_Date[wbr.Coupe] = wbr.Date;
}
if (!Operations_set.has(wbr.Operation) && wbr.Operation && wbr.Operation != 'TEST') {
//console.log(wbr.Operation,wbr.Date);
new_Operations.add(wbr.Operation);
new_Operations_Date[wbr.Operation] = wbr.Date;
}
if (!Harvestor_set.has(wbr.Harvestor) && wbr.Harvestor && wbr.Harvestor != 'TEST') new_Harvestor.add(wbr.Harvestor);
if (!Supplier_set.has(wbr.Supplier) && wbr.Supplier && wbr.Supplier != 'TEST') new_Supplier.add(wbr.Supplier);
if (!FPP_set.has(wbr.FPP) && wbr.FPP && wbr.FPP != 'TEST' && wbr.FPP != "18-34") {
new_FPP.add(wbr.FPP);
new_FPPExpiry[wbr.FPP] = wbr.Expiry;
}
//console.log(wbr.Docket);
if (acc.wb.length > 0 && acc.wb[acc.wb.length - 1][0] === wbr.Docket) {
const n = acc.wb.length - 1;
//console.log("Rear Load Found",acc.wb[n][8]);
acc.wb[n][9] = wbr.Gross + acc.wb[n][9];
acc.wb[n][10] = wbr.Tare + acc.wb[n][10];
acc.wb[n][11] = wbr.GMT + acc.wb[n][11];
} else if (acc.iv_wb.length > 0 && acc.wb[acc.iv_wb.length - 1][0] === wbr.Docket) {
const n = acc.iv_wb.length - 1;
acc.iv_wb[n][9] = wbr.Gross + acc.iv_wb[n][10];
acc.iv_wb[n][10] = wbr.Tare + acc.iv_wb[n][11];
acc.iv_wb[n][11] = wbr.GMT + acc.iv_wb[n][12];
//console.log("Invalid Rear Load Found");
} else if (!Docket_set.has(wbr.Docket)) {
if (new_Dockets.has(wbr.Docket)) {
invalid(" Duplicate Docket - only added first instance");
} else if ((isNaN(wbr.Docket)) || (wbr.Docket < 9999) || (wbr.Docket > 9999999)) {
invalid(" Check Docket Number ");
console.log(wbr.Docket);
} else if ((wbr.Coupe === 'TEST') || (wbr.Supplier === 'TEST') || (wbr.Harvestor === 'TEST') || (wbr.Operation === 'TEST' || wbr.FPP === 'TEST')) {
invalid(" Contains Test Values");
} else if (!wbr.Coupe || !wbr.Supplier || !wbr.Harvestor || !wbr.Operation || !wbr.FPP || wbr.FPP === "18-34") {
invalid(" Contains Null Values");
} else if (!wbr.GMT || !wbr.Tare || !wbr.Gross) {
invalid(" Contains Null Weight Values");
} else if (!Grade_set.has(wbr.Grade)) {
invalid(" Check Grade and add to Grades Table if valid");
} else if (!Len_set.has(wbr.Length)) {
console.log(wbr.Length, Len_set);
invalid(" Check Length and add to Lengths Table if valid");
} else {
//console.log("Valid");
valid();
new_Dockets.add(wbr.Docket);
}
} else {
showAlert("Duplicate Docket - Ignoring" + wbr.Docket.toString(), 'error');
}
return acc;
},
{
wb: [],
iv_wb: [],
}
);
//console.log('finished raw processing')
if (new_Coupe.size) {
const coupe_data = Array.from(new_Coupe).map(item => `('${item}','${new_Coupe_Date[item]}')`).join(',');
console.log("New Coupe: ", coupe_data);
Insert_Coupe.run({'data': coupe_data});
}
if (new_Operations.size) {
const operation_data = Array.from(new_Operations).map(item => `('${item}','${new_Operations_Date[item]}')`).join(',');
console.log("New Operation: ", operation_data);
Insert_Operations.run({'data': operation_data});
}
if (new_Harvestor.size) {
const harvestor_data = Array.from(new_Harvestor).map(item => `('${item}')`).join(',');
console.log("New Harvestor: ", harvestor_data);
Insert_Harvestor.run({'data': harvestor_data});
}
if (new_Supplier.size) {
const supplier_data = Array.from(new_Supplier).map(item => `('${item}')`).join(',');
console.log("New Supplier", supplier_data);
Insert_Supplier.run({'data': supplier_data});
}
if (new_FPP.size) {
const fpp_data = Array.from(new_FPP).map(item => `('${item}','${new_FPPExpiry[item]}')`).join(',').replace(/''/g, 'NULL');
console.log("New FPP", fpp_data);
Insert_FPP.run({'data': fpp_data});
}
console.log('inserting');
if (wb.length > 0) {
const data = wb.map(item => `(${item.map(val => `'${val}'`).join(',')})`).join(',');
Insert_WeighBridge.run({'data': data})
.then(() => showAlert("Found Valid WeighBridge Entires", 'success'))
.catch((error) => showAlert(error, 'error'));
}
if (iv_wb.length > 0) {
const idata = iv_wb.map(item => `(${item.map(val => `'${val}'`).join(',')})`).join(',');
Insert_Invalid_WeighBridge.run({'data': idata})
.then(() => showAlert("Found Invalid WeighBridge Entires", 'success'))
.catch((error) => showAlert(error, 'error'));
}
await WB_Select.run();
await IW_Select.run();
Count_Rows.run();
//console.Docket(iv_wb_data);
} catch (error) {
showAlert(error, 'error')
}
}
}