diff --git a/app/controllers/products.controller.js b/app/controllers/products.controller.js index 2cc1505..2c50573 100644 --- a/app/controllers/products.controller.js +++ b/app/controllers/products.controller.js @@ -217,7 +217,12 @@ const filePath = uploadedPath; const cleanRow = {}; for (const key in row) { - const normalizedKey = key.replace(/[\s\W]+/g, "_").trim().toLowerCase(); + let cleanedHeader = key + .replace(/\*/g, "") + .replace(/\(.*?\)/g, "") + .trim(); + + const normalizedKey = cleanedHeader.replace(/[\s\W]+/g, "_").trim().toLowerCase(); let mappedKey = normalizedKey; if (/^hs.*code$/.test(normalizedKey) || normalizedKey === "hscode") { @@ -281,7 +286,10 @@ const filePath = uploadedPath; errors.push({ row: index + 1, error: "HS Code must be max 10 digits" }); continue; } - + if (productName.length > 1000) { + errors.push({ row: index + 1, error: "Product Name is too long. Maximum allowed length is 1000 characters." }); + continue; + } if (description.length > 1000) { errors.push({ row: index + 1, error: "HS Description is too long. Maximum allowed length is 1000 characters." }); continue; diff --git a/app/controllers/unitMasterController.js b/app/controllers/unitMasterController.js index 551d187..f4d7c50 100644 --- a/app/controllers/unitMasterController.js +++ b/app/controllers/unitMasterController.js @@ -209,7 +209,7 @@ exports.uploadUnitMasterFromCSV = async (req, res) => { }); } - const results = []; + let results = []; fs.createReadStream(filePath) .pipe(csv()) @@ -224,24 +224,68 @@ exports.uploadUnitMasterFromCSV = async (req, res) => { }); } - //Expected columns - const requiredCols = ["Unit Name", "Description"]; - const headers = Object.keys(results[0] || {}); - const missingCols = requiredCols.filter((col) => !headers.includes(col)); - const extraCols = headers.filter((col) => !requiredCols.includes(col)); + // ------------------------------------------------------------------- + // NORMALIZE HEADERS (Very Important!) + // ------------------------------------------------------------------- + const normalizeHeader = (h) => { + return h + .replace(/\*/g, "") // remove * + .replace(/\(.*?\)/g, "") // remove (Mandatory) + .trim() + .replace(/[\s\W]+/g, "_") // spaces & special chars -> _ + .toLowerCase(); + }; + + // Required normalized fields + const mappedRequiredCols = { + unit_name: "Unit Name", + description: "Description", + }; + + const incomingHeaders = Object.keys(results[0] || {}); + const normalizedIncoming = incomingHeaders.map(h => normalizeHeader(h)); + + // Check missing columns + const missingCols = Object.keys(mappedRequiredCols).filter( + req => !normalizedIncoming.includes(req) + ); + + // Check unexpected columns + const extraCols = normalizedIncoming.filter( + col => !Object.keys(mappedRequiredCols).includes(col) + ); if (missingCols.length > 0 || extraCols.length > 0) { fs.unlinkSync(filePath); + let msg = ""; if (missingCols.length > 0) - msg += `Missing required columns: ${missingCols.join(", ")}. `; + msg += `Missing required columns: ${missingCols.map(c => mappedRequiredCols[c]).join(", ")}. `; + if (extraCols.length > 0) - msg += `Unexpected columns found: ${extraCols.join( - ", " - )}. Only 'Unit Name' and 'Description' are allowed.`; - return res.status(400).send({ status: "failed", message: msg.trim() }); + msg += `Unexpected columns found: ${extraCols.join(", ")}. Only 'Unit Name' and 'Description' are allowed.`; + + return res.status(400).send({ + status: "failed", + message: msg.trim(), + }); } + // Remap row keys to clean headers + results = results.map(row => { + const newRow = {}; + for (const key in row) { + const normalized = normalizeHeader(key); + const mapped = mappedRequiredCols[normalized]; + if (mapped) newRow[mapped] = row[key]; + } + return newRow; + }); + + // ------------------------------------------------------------------- + // VALIDATION AND PROCESSING + // ------------------------------------------------------------------- + const inserted = []; const duplicates = []; const errors = []; @@ -251,7 +295,7 @@ exports.uploadUnitMasterFromCSV = async (req, res) => { const generateShortName = (unitName, existingShorts = new Set()) => { const cleaned = unitName.replace(/[^a-zA-Z]/g, "").toUpperCase(); - // Predefined abbreviations + const abbreviationMap = { METER: "MT", METRE: "MT", @@ -272,54 +316,59 @@ exports.uploadUnitMasterFromCSV = async (req, res) => { UNIT: "UNT", }; - //if Unit Name is ≤ 5 characters, just use it in uppercase if (cleaned.length <= 5) return cleaned; - // Otherwise, use abbreviation map or generate automatically + let shortName = abbreviationMap[cleaned] || cleaned.substring(0, 3); if (shortName.length < 2) shortName = shortName.padEnd(2, "X"); - // Random 2-letter suffix for uniqueness + const randomLetters = () => Array.from({ length: 2 }, () => String.fromCharCode(65 + Math.floor(Math.random() * 26)) ).join(""); - let finalShort = `${shortName}${randomLetters()}`; - while (existingShorts.has(finalShort.toLowerCase())) { - finalShort = `${shortName}${randomLetters()}`; + let final = `${shortName}${randomLetters()}`; + while (existingShorts.has(final.toLowerCase())) { + final = `${shortName}${randomLetters()}`; } - return finalShort.substring(0, 5); + + return final.substring(0, 5); }; - //Validate & prepare records - for (const [index, row] of results.entries()) { + // Row validation + for (const [i, row] of results.entries()) { const uom = row["Unit Name"]?.trim(); const description = row["Description"]?.trim() || null; + if (description.length > 1000) { + errors.push({ row: i + 1, reason: "HS Description is too long. Maximum allowed length is 1000 characters." }); + continue; + } + if (!uom) { - errors.push({ row: index + 1, reason: "Missing 'Unit Name'." }); + errors.push({ row: i + 1, reason: "Missing 'Unit Name'." }); continue; } const uomKey = uom.toLowerCase(); if (seenUnitNames.has(uomKey)) { fileDuplicates.push({ - row: index + 1, + row: i + 1, reason: "Duplicate Unit Name in file.", uom, }); continue; } - let uomShort = generateShortName(uom, seenShortNames); + const uomShort = generateShortName(uom, seenShortNames); seenUnitNames.add(uomKey); seenShortNames.add(uomShort.toLowerCase()); - results[index]._generatedShort = uomShort; - results[index]._description = description; + + results[i]._generatedShort = uomShort; + results[i]._description = description; } - // Stop if file has internal duplicates if (fileDuplicates.length > 0) { fs.unlinkSync(filePath); return res.status(400).send({ @@ -329,10 +378,10 @@ exports.uploadUnitMasterFromCSV = async (req, res) => { }); } - //Insert into DB with safe duplicate handling - for (const [index, row] of results.entries()) { + // Insert into DB + for (const [i, row] of results.entries()) { try { - const uom = row["Unit Name"]?.trim(); + const uom = row["Unit Name"].trim(); let uomShort = row._generatedShort; const description = row._description; @@ -352,34 +401,12 @@ exports.uploadUnitMasterFromCSV = async (req, res) => { }); if (existing) { - // Skip if same Unit Name already exists - if (existing.uom.toLowerCase() === uom.toLowerCase()) { - duplicates.push({ - id: existing.id, - uom: existing.uom, - uom_short_name: existing.uom_short_name, - }); - continue; - } - - //Safe regeneration using ternary inside while loop - let newShort = uomShort; - let existsInDb; - - do { - existsInDb = await UnitMaster.findOne({ - where: Sequelize.where( - Sequelize.fn("LOWER", Sequelize.col("uom_short_name")), - newShort.toLowerCase() - ), - }); - - newShort = existsInDb - ? `${uomShort}${Math.floor(Math.random() * 1000)}`.substring(0, 5) - : newShort; - } while (existsInDb); - - uomShort = newShort; + duplicates.push({ + id: existing.id, + uom: existing.uom, + uom_short_name: existing.uom_short_name, + }); + continue; } const newUnit = await UnitMaster.create({ @@ -391,13 +418,18 @@ exports.uploadUnitMasterFromCSV = async (req, res) => { }); inserted.push(newUnit); + } catch (err) { - errors.push({ row: index + 1, reason: err.message }); + errors.push({ row: i + 1, reason: err.message }); } } fs.unlinkSync(filePath); + // ------------------------------------------------------------------- + // FINAL RESPONSE + // ------------------------------------------------------------------- + let finalStatus = "success"; let message = `${inserted.length} units inserted successfully.`; @@ -421,6 +453,7 @@ exports.uploadUnitMasterFromCSV = async (req, res) => { duplicates, error_details: errors, }); + } catch (err) { if (fs.existsSync(filePath)) fs.unlinkSync(filePath); return res.status(500).send({ @@ -431,7 +464,9 @@ exports.uploadUnitMasterFromCSV = async (req, res) => { }); } catch (error) { console.error("Error uploading Unit Master CSV:", error); - if (req.file && fs.existsSync(path.resolve(req.file.path))) fs.unlinkSync(path.resolve(req.file.path)); + if (req.file && fs.existsSync(path.resolve(req.file.path))) + fs.unlinkSync(path.resolve(req.file.path)); + return res.status(500).send({ status: "failed", message: "Error processing CSV file.", @@ -440,15 +475,6 @@ exports.uploadUnitMasterFromCSV = async (req, res) => { } }; -// exports.downloadUnitMasterFile = async (req, res) => { -// try { -// const filePath = path.join(__dirname, "../uploads/unit_master_sample.csv"); -// return res.download(filePath, "unit_master_sample.csv"); -// } catch (error) { -// return res.status(500).send({ status: "failed", message: error.message }); -// } -// }; - exports.downloadUnitMasterFile = async (req, res) => { try { const safeBasePath = path.resolve(__dirname, "../downloads_csv"); diff --git a/app/downloads_csv/products_upload_sample.csv b/app/downloads_csv/products_upload_sample.csv index 9ad3957..fa2ee9e 100644 --- a/app/downloads_csv/products_upload_sample.csv +++ b/app/downloads_csv/products_upload_sample.csv @@ -1 +1 @@ -HS Code,Product Name,Unit,Description +HS Code * (Mandatory),Product Name * (Mandatory),Unit * (Mandatory),Description diff --git a/app/downloads_csv/unit_master_sample.csv b/app/downloads_csv/unit_master_sample.csv index b2dd7cd..21af918 100644 --- a/app/downloads_csv/unit_master_sample.csv +++ b/app/downloads_csv/unit_master_sample.csv @@ -1 +1 @@ -Unit Name,Description +Unit Name * (Mandatory),Description * (Mandatory) diff --git a/app/models/UnitMaster.model.js b/app/models/UnitMaster.model.js index c7647e7..7a0825b 100644 --- a/app/models/UnitMaster.model.js +++ b/app/models/UnitMaster.model.js @@ -31,6 +31,11 @@ module.exports = (sequelize, DataTypes) => { description: { type: DataTypes.STRING, allowNull: true, + validate: { + len: { + args: [0, 1000], + msg: "Description is too long. Maximum allowed length is 1000 characters." + }} }, is_active: { type: DataTypes.BOOLEAN,