diff --git a/app/controllers/products.controller.js b/app/controllers/products.controller.js index dd8e24a..d954c48 100644 --- a/app/controllers/products.controller.js +++ b/app/controllers/products.controller.js @@ -43,6 +43,105 @@ function parseWeightInIb(value) { return { ok: true, value: rounded }; } +/** Map unit_master by full name (uom) and short code (uom_short_name), case-insensitive. */ +function buildUnitLookupMap(unitMasters) { + const unitMap = new Map(); + for (const unit of unitMasters) { + if (unit.uom) { + unitMap.set(unit.uom.trim().toLowerCase(), unit.id); + } + if (unit.uom_short_name) { + unitMap.set(unit.uom_short_name.trim().toLowerCase(), unit.id); + } + } + return unitMap; +} + +function normalizeCsvHeaderKey(key) { + if (!key) return ""; + const withoutBom = String(key).replace(/^\uFEFF/, ""); + return withoutBom + .replace(/\*/g, "") + .replace(/\(.*?\)/g, "") + .trim() + .replace(/[\s\W]+/g, "_") + .trim() + .toLowerCase(); +} + +function mapCsvRowKey(normalizedKey) { + if (/^hs.*code$/.test(normalizedKey) || normalizedKey === "hscode") { + return "hs_code"; + } + if (/^product.*name$/.test(normalizedKey)) { + return "product_name"; + } + if ( + normalizedKey === "unit" || + normalizedKey === "uom" || + normalizedKey === "measurement_unit" || + normalizedKey === "measure" || + normalizedKey.endsWith("_unit") + ) { + return "unit"; + } + if (/^desc(ription)?$/.test(normalizedKey)) { + return "description"; + } + if ( + normalizedKey === "weight_in_ib" || + normalizedKey === "weight_in_lb" || + /^weight.*(ib|lb)/i.test(normalizedKey) || + normalizedKey === "weight" + ) { + return "weight_in_ib"; + } + return normalizedKey; +} + +function transformCsvRow(row) { + const cleanRow = {}; + for (const key in row) { + if (!Object.prototype.hasOwnProperty.call(row, key)) continue; + const mappedKey = mapCsvRowKey(normalizeCsvHeaderKey(key)); + const value = + row[key] !== undefined && row[key] !== null && String(row[key]).trim() !== "" + ? String(row[key]).trim() + : null; + if (cleanRow[mappedKey] && !value) continue; + cleanRow[mappedKey] = value; + } + return cleanRow; +} + +function isBlankCsvRow(row) { + return ( + !row.hs_code && + !row.product_name && + !row.unit && + !row.description && + (row.weight_in_ib === undefined || + row.weight_in_ib === null || + String(row.weight_in_ib).trim() === "") + ); +} + +function parseCsvFile(filePath) { + return new Promise((resolve, reject) => { + const results = []; + fs.createReadStream(filePath) + .pipe(csv()) + .on("data", (row) => results.push(transformCsvRow(row))) + .on("end", () => resolve(results)) + .on("error", reject); + }); +} + +function sendUploadResponse(res, statusCode, body) { + if (res.headersSent) return; + return res.status(statusCode).json(body); +} + /** Returns true if a response was sent (4xx). */ function tryRespondProductPersistenceError(error, res) { if (!error || res.headersSent) { @@ -334,420 +433,376 @@ exports.uploadProductsFromCSV = async (req, res) => { let sanitizedPath = null; try { - // Validate file upload if (!req.file) { - return res.status(400).send({ - status: "failed", - message: "No file uploaded" + return sendUploadResponse(res, 400, { + status: "failed", + message: "No file uploaded", }); } - + try { sanitizedPath = sanitizeFilePath(req.file.path, UPLOAD_DIR); } catch (sanitizeError) { - // Attempt cleanup with original path if sanitization fails try { const unsafePath = req.file.path; if (unsafePath && fs.existsSync(unsafePath)) { fs.unlinkSync(unsafePath); } } catch (cleanupErr) { - // Silent fail on cleanup + // ignore cleanup errors } - - return res.status(400).send({ + return sendUploadResponse(res, 400, { status: "failed", - message: "Invalid file path detected" + message: "Invalid file path detected", }); } - // Validate file exists after sanitization if (!validateFileExists(sanitizedPath)) { - return res.status(400).send({ + return sendUploadResponse(res, 400, { status: "failed", - message: "File not found after validation" + message: "File not found after validation", }); } + const originalName = path.basename(req.file.originalname); const fileExt = path.extname(originalName).toLowerCase(); - - if (fileExt !== '.csv') { - deleteFileSecure(sanitizedPath); - return res.status(400).send({ - status: "failed", - message: "Invalid file type. Only CSV allowed." - }); - } - // Validate MIME type - const allowedMimes = ['text/csv', 'application/csv', 'text/plain']; - if (req.file.mimetype && !allowedMimes.some(mime => mime === req.file.mimetype)) { + if (fileExt !== ".csv") { deleteFileSecure(sanitizedPath); - return res.status(400).send({ + return sendUploadResponse(res, 400, { status: "failed", - message: "Invalid MIME type. Only CSV allowed." + message: "Invalid file type. Only CSV allowed.", }); } - // Validate user ID - if (!req.user || !req.user.id || isNaN(req.user.id)) { + const allowedMimes = ["text/csv", "application/csv", "text/plain", "application/vnd.ms-excel"]; + if (req.file.mimetype && !allowedMimes.includes(req.file.mimetype)) { deleteFileSecure(sanitizedPath); - return res.status(400).send({ - status: "failed", - message: "Invalid or missing User Id." + return sendUploadResponse(res, 400, { + status: "failed", + message: "Invalid MIME type. Only CSV allowed.", + }); + } + + if (!req.user || !req.user.id || Number.isNaN(Number(req.user.id))) { + deleteFileSecure(sanitizedPath); + return sendUploadResponse(res, 400, { + status: "failed", + message: "Invalid or missing User Id.", }); } - // Check file size const fileStats = fs.statSync(sanitizedPath); if (fileStats.size === 0) { deleteFileSecure(sanitizedPath); - return res.status(400).send({ + return sendUploadResponse(res, 400, { status: "failed", message: "Uploaded file is empty.", }); } - const csvResults = []; const userId = parseInt(req.user.id, 10); + const csvResults = await parseCsvFile(sanitizedPath); - // Process CSV using sanitized path - fs.createReadStream(sanitizedPath) - .pipe(csv()) - .on("data", (row) => { - const cleanRow = {}; - - for (const key in row) { - if (!row.hasOwnProperty(key)) continue; - - let cleanedHeader = key - .replace(/\*/g, "") - .replace(/\(.*?\)/g, "") - .trim(); - - const normalizedKey = cleanedHeader - .replace(/[\s\W]+/g, "_") - .trim() - .toLowerCase(); - - let mappedKey = normalizedKey; - if (/^hs.*code$/.test(normalizedKey) || normalizedKey === "hscode") { - mappedKey = "hs_code"; - } else if (/^product.*name$/.test(normalizedKey)) { - mappedKey = "product_name"; - } else if (/unit|measurement|uom|measure/i.test(normalizedKey)) { - mappedKey = "unit"; - } else if (/desc(ription)?/i.test(normalizedKey)) { - mappedKey = "description"; - } else if ( - normalizedKey === "weight_in_ib" || - /^weight.*(ib|lb)/i.test(normalizedKey) || - /^weight$/i.test(normalizedKey) - ) { - mappedKey = "weight_in_ib"; - } - - cleanRow[mappedKey] = row[key] ? row[key].trim() : null; - } - - csvResults.push(cleanRow); - }) - .on("end", async () => { - try { - // Validate CSV has data - if (csvResults.length === 0) { - deleteFileSecure(sanitizedPath); - return res.status(400).send({ - status: "failed", - message: "CSV file is empty or invalid." - }); - } - - // Validate required columns - const requiredCols = ["hs_code", "product_name", "unit", "description"]; - const optionalCols = ["weight_in_ib"]; - const allowedCols = requiredCols.concat(optionalCols); - const headers = Object.keys(csvResults[0]); - - const missingCols = requiredCols.filter(col => !headers.includes(col)); - const extraCols = headers.filter(col => !allowedCols.includes(col)); - - if (missingCols.length > 0 || extraCols.length > 0) { - deleteFileSecure(sanitizedPath); - return res.status(400).send({ - status: "failed", - message: - (missingCols.length ? `Missing columns: ${missingCols.join(", ")}. ` : "") + - (extraCols.length ? `Unexpected columns: ${extraCols.join(", ")}.` : "") - }); - } - - // Process and validate rows - const normalizedRows = []; - const seenHsCodes = new Set(); - const fileDuplicates = new Set(); - const validationErrors = []; - - for (let index = 0; index < csvResults.length; index++) { - const row = csvResults[index]; - const rowNumber = index + 1; - - let hsCode = row.hs_code ? row.hs_code.replace(/[-\s/]/g, "").trim() : ""; - const productName = row.product_name ? row.product_name.replace(/\s+/g, " ").trim() : ""; - const unit = row.unit ? row.unit.trim().toLowerCase() : ""; - const description = row.description ? row.description.trim().toLowerCase() : ""; - - let weightInIb = null; - if ( - row.weight_in_ib !== undefined && - row.weight_in_ib !== null && - String(row.weight_in_ib).trim() !== "" - ) { - const wp = parseWeightInIb(row.weight_in_ib); - if (!wp.ok) { - validationErrors.push({ - row: rowNumber, - error: wp.message, - }); - continue; - } - weightInIb = wp.value; - } - - // Validate required fields - if (!hsCode || !productName) { - validationErrors.push({ - row: rowNumber, - error: "Missing required HS Code or Product Name" - }); - continue; - } - - // Validate HS code format - if (!/^\d+$/.test(hsCode)) { - validationErrors.push({ - row: rowNumber, - error: "HS Code must be numeric" - }); - continue; - } - - // Validate HS code length - if (hsCode.length > 10) { - validationErrors.push({ - row: rowNumber, - error: "HS Code must be max 10 digits" - }); - continue; - } - - // Validate product name length - if (productName.length > 1000) { - validationErrors.push({ - row: rowNumber, - error: "Product Name is too long. Maximum 1000 characters." - }); - continue; - } - - // Validate description length - if (description.length > 1000) { - validationErrors.push({ - row: rowNumber, - error: "HS Description is too long. Maximum 1000 characters." - }); - continue; - } - - // Check for duplicates in file - const normalizedHs = hsCode.replace(/^0+/, ""); - if (seenHsCodes.has(normalizedHs)) { - fileDuplicates.add(hsCode); - continue; - } - seenHsCodes.add(normalizedHs); - - normalizedRows.push({ - hsCode: hsCode, - productName: productName, - unit: unit, - description: description, - weightInIb, - }); - } - - // Check for file duplicates - if (fileDuplicates.size > 0) { - deleteFileSecure(sanitizedPath); - return res.status(400).send({ - status: "failed", - message: "Duplicate HS Codes found within file. Resolve and re-upload.", - duplicate_hs_codes_in_file: Array.from(fileDuplicates) - }); - } - - const unitMasters = await UnitMaster.findAll({ - attributes: ["id", "uom"] - }); - - const unitMap = new Map(); - for (let i = 0; i < unitMasters.length; i++) { - const unit = unitMasters[i]; - const key = unit.uom.trim().toLowerCase(); - unitMap.set(key, unit.id); - } - - const existingProducts = await Product.findAll({ - attributes: ["hs_code", "product_name"] - }); - - const existingHsCodeSet = new Set(); - const existingProductNameSet = new Set(); - for (let i = 0; i < existingProducts.length; i++) { - const normalized = existingProducts[i].hs_code.replace(/^0+/, ""); - existingHsCodeSet.add(normalized); - - const productNameLower = existingProducts[i].product_name.trim().toLowerCase(); - existingProductNameSet.add(productNameLower); - } - - const duplicateHsCodes = []; - const duplicateProductNames = []; - - for (let i = 0; i < normalizedRows.length; i++) { - const row = normalizedRows[i]; - const normalizedHs = row.hsCode.replace(/^0+/, ""); - const productNameLower = row.productName.trim().toLowerCase(); - - if (existingHsCodeSet.has(normalizedHs)) { - duplicateHsCodes.push({ - row: i + 1, - hs_code: row.hsCode, - product_name: row.productName - }); - } - - if (existingProductNameSet.has(productNameLower)) { - duplicateProductNames.push({ - row: i + 1, - hs_code: row.hsCode, - product_name: row.productName - }); - } - } - - if (duplicateHsCodes.length > 0 || duplicateProductNames.length > 0) { - deleteFileSecure(sanitizedPath); - - const errorMessages = []; - - if (duplicateHsCodes.length > 0) { - errorMessages.push(`${duplicateHsCodes.length} duplicate HS Code(s) found in database`); - } - - if (duplicateProductNames.length > 0) { - errorMessages.push(`${duplicateProductNames.length} duplicate Product Name(s) found in database`); - } - - return res.status(400).send({ - status: "failed", - message: "Upload rejected: " + errorMessages.join(", ") + ". Please remove duplicates and try again.", - duplicate_hs_codes: duplicateHsCodes, - duplicate_product_names: duplicateProductNames, - total_duplicates: duplicateHsCodes.length + duplicateProductNames.length - }); - } - - const toInsert = []; - - for (let i = 0; i < normalizedRows.length; i++) { - const row = normalizedRows[i]; - - // Validate unit - const unitId = unitMap.get(row.unit); - if (!unitId) { - validationErrors.push({ - row: i + 1, - error: "Invalid unit: " + row.unit - }); - continue; - } - - toInsert.push({ - hs_code: row.hsCode, - product_name: row.productName, - unit_id: unitId, - hs_description: row.description, - weight_in_ib: row.weightInIb, - created_by: userId, - created_at: new Date(), - }); - } - - if (validationErrors.length > 0) { - deleteFileSecure(sanitizedPath); - return res.status(400).send({ - status: "failed", - message: `Upload rejected: ${validationErrors.length} validation error(s) found. Please fix and try again.`, - errors: validationErrors - }); - } - - let insertedRecords = []; - if (toInsert.length > 0) { - insertedRecords = await Product.bulkCreate(toInsert, { - validate: true - }); - } - - deleteFileSecure(sanitizedPath); - - return res.status(200).send({ - status: "success", - message: `${insertedRecords.length} products inserted successfully.`, - summary: { - total_records: csvResults.length, - imported: insertedRecords.length, - skipped: 0, - errors: [] - } - }); - - } catch (processingError) { - deleteFileSecure(sanitizedPath); - if (logger && logger.error) { - logger.error("CSV processing error: " + processingError.message); - logger.error(`Stack trace: ${processingError.stack}`); - } - if (tryRespondProductPersistenceError(processingError, res)) { - return; - } - return res.status(500).send({ - status: "failed", - message: "Internal server error", - }); - } - }) - .on("error", (streamError) => { - deleteFileSecure(sanitizedPath); - if (logger && logger.error) { - logger.error("Stream error: " + streamError.message); - } - return res.status(500).send({ - status: "failed", - message: "Error reading CSV file" - }); + if (csvResults.length === 0) { + deleteFileSecure(sanitizedPath); + return sendUploadResponse(res, 400, { + status: "failed", + message: "CSV file is empty or invalid.", }); + } + const requiredCols = ["hs_code", "product_name", "unit"]; + const optionalCols = ["description", "weight_in_ib"]; + const allowedCols = requiredCols.concat(optionalCols); + const headers = Object.keys(csvResults[0]); + + const missingCols = requiredCols.filter((col) => !headers.includes(col)); + const extraCols = headers.filter((col) => !allowedCols.includes(col)); + + if (missingCols.length > 0 || extraCols.length > 0) { + deleteFileSecure(sanitizedPath); + return sendUploadResponse(res, 400, { + status: "failed", + message: + (missingCols.length ? `Missing columns: ${missingCols.join(", ")}. ` : "") + + (extraCols.length ? `Unexpected columns: ${extraCols.join(", ")}.` : ""), + }); + } + + const normalizedRows = []; + const seenHsCodes = new Set(); + const fileDuplicates = new Set(); + const validationErrors = []; + + for (let index = 0; index < csvResults.length; index++) { + const row = csvResults[index]; + const csvLineNumber = index + 2; + + if (isBlankCsvRow(row)) { + continue; + } + + const hsCode = row.hs_code ? row.hs_code.replace(/[-\s/]/g, "").trim() : ""; + const productName = row.product_name + ? row.product_name.replace(/\s+/g, " ").trim() + : ""; + const unit = row.unit ? row.unit.trim().toLowerCase() : ""; + const description = row.description ? row.description.trim() : ""; + + let weightInIb = null; + if ( + row.weight_in_ib !== undefined && + row.weight_in_ib !== null && + String(row.weight_in_ib).trim() !== "" + ) { + const wp = parseWeightInIb(row.weight_in_ib); + if (!wp.ok) { + validationErrors.push({ row: csvLineNumber, error: wp.message }); + continue; + } + weightInIb = wp.value; + } + + if (!hsCode || !productName || !unit) { + const missing = []; + if (!hsCode) missing.push("HS Code"); + if (!productName) missing.push("Product Name"); + if (!unit) missing.push("Unit"); + validationErrors.push({ + row: csvLineNumber, + error: `Missing required field(s): ${missing.join(", ")}`, + }); + continue; + } + + if (!/^\d+$/.test(hsCode)) { + validationErrors.push({ row: csvLineNumber, error: "HS Code must be numeric" }); + continue; + } + + if (hsCode.length > 10) { + validationErrors.push({ + row: csvLineNumber, + error: "HS Code must be max 10 digits", + }); + continue; + } + + if (hsCode === "0" || /^0+$/.test(hsCode)) { + validationErrors.push({ + row: csvLineNumber, + error: "HS Code cannot be 0", + }); + continue; + } + + if (productName.length > 1000) { + validationErrors.push({ + row: csvLineNumber, + error: "Product Name is too long. Maximum 1000 characters.", + }); + continue; + } + + if (description.length > 1000) { + validationErrors.push({ + row: csvLineNumber, + error: "HS Description is too long. Maximum 1000 characters.", + }); + continue; + } + + const normalizedHs = hsCode.replace(/^0+/, "") || hsCode; + if (seenHsCodes.has(normalizedHs)) { + fileDuplicates.add(hsCode); + continue; + } + seenHsCodes.add(normalizedHs); + + normalizedRows.push({ + hsCode, + productName, + unit, + description, + weightInIb, + csvLineNumber, + }); + } + + if (normalizedRows.length === 0 && validationErrors.length === 0) { + deleteFileSecure(sanitizedPath); + return sendUploadResponse(res, 400, { + status: "failed", + message: + "No product data rows found. Add data below the header row (remove blank rows from the template).", + }); + } + + if (fileDuplicates.size > 0) { + deleteFileSecure(sanitizedPath); + return sendUploadResponse(res, 400, { + status: "failed", + message: + "Duplicate HS Codes found within file. Resolve and re-upload.", + duplicate_hs_codes_in_file: Array.from(fileDuplicates), + }); + } + + const unitMasters = await UnitMaster.findAll({ + attributes: ["id", "uom", "uom_short_name"], + where: { is_active: true }, + }); + const unitMap = buildUnitLookupMap(unitMasters); + + const existingProducts = await Product.findAll({ + attributes: ["hs_code", "product_name"], + where: { is_active: true }, + }); + + const existingHsCodeSet = new Set(); + const existingProductNameSet = new Set(); + for (const product of existingProducts) { + if (product.hs_code != null && String(product.hs_code).trim() !== "") { + existingHsCodeSet.add(String(product.hs_code).replace(/^0+/, "") || "0"); + } + if (product.product_name) { + existingProductNameSet.add(product.product_name.trim().toLowerCase()); + } + } + + const duplicateHsCodes = []; + const duplicateProductNames = []; + + for (let i = 0; i < normalizedRows.length; i++) { + const row = normalizedRows[i]; + const normalizedHs = row.hsCode.replace(/^0+/, "") || row.hsCode; + const productNameLower = row.productName.trim().toLowerCase(); + + if (existingHsCodeSet.has(normalizedHs)) { + duplicateHsCodes.push({ + row: row.csvLineNumber, + hs_code: row.hsCode, + product_name: row.productName, + }); + } + + if (existingProductNameSet.has(productNameLower)) { + duplicateProductNames.push({ + row: row.csvLineNumber, + hs_code: row.hsCode, + product_name: row.productName, + }); + } + } + + if (duplicateHsCodes.length > 0 || duplicateProductNames.length > 0) { + deleteFileSecure(sanitizedPath); + const errorMessages = []; + if (duplicateHsCodes.length > 0) { + errorMessages.push( + `${duplicateHsCodes.length} duplicate HS Code(s) found in database` + ); + } + if (duplicateProductNames.length > 0) { + errorMessages.push( + `${duplicateProductNames.length} duplicate Product Name(s) found in database` + ); + } + return sendUploadResponse(res, 400, { + status: "failed", + message: + "Upload rejected: " + + errorMessages.join(", ") + + ". Please remove duplicates and try again.", + duplicate_hs_codes: duplicateHsCodes, + duplicate_product_names: duplicateProductNames, + total_duplicates: + duplicateHsCodes.length + duplicateProductNames.length, + }); + } + + const toInsert = []; + + for (let i = 0; i < normalizedRows.length; i++) { + const row = normalizedRows[i]; + const unitId = unitMap.get(row.unit); + + if (!unitId) { + validationErrors.push({ + row: row.csvLineNumber, + error: `Invalid unit "${row.unit}". Use unit short code (e.g. NO) or full name (e.g. Numbers) from Unit Master.`, + }); + continue; + } + + toInsert.push({ + hs_code: parseInt(row.hsCode, 10), + product_name: row.productName, + unit_id: unitId, + hs_description: row.description || null, + weight_in_ib: row.weightInIb, + created_by: userId, + created_at: new Date(), + is_active: true, + }); + } + + if (validationErrors.length > 0) { + deleteFileSecure(sanitizedPath); + return sendUploadResponse(res, 400, { + status: "failed", + message: `Upload rejected: ${validationErrors.length} validation error(s) found. Please fix and try again.`, + errors: validationErrors, + }); + } + + if (toInsert.length === 0) { + deleteFileSecure(sanitizedPath); + return sendUploadResponse(res, 400, { + status: "failed", + message: "No valid product rows to import.", + }); + } + + let insertedRecords = []; + try { + insertedRecords = await Product.bulkCreate(toInsert, { + validate: true, + individualHooks: false, + }); + } catch (bulkError) { + deleteFileSecure(sanitizedPath); + logger.error("Product bulkCreate error: " + bulkError.message); + logger.error(`Stack trace: ${bulkError.stack}`); + if (tryRespondProductPersistenceError(bulkError, res)) { + return; + } + throw bulkError; + } + + deleteFileSecure(sanitizedPath); + + return sendUploadResponse(res, 201, { + status: "success", + message: `${insertedRecords.length} product(s) inserted successfully.`, + summary: { + total_records: csvResults.length, + imported: insertedRecords.length, + skipped: csvResults.length - insertedRecords.length, + errors: [], + }, + }); } catch (error) { deleteFileSecure(sanitizedPath); - if (logger && logger.error) { - logger.error("Upload error: " + error.message); - logger.error(`Stack trace: ${error.stack}`); + logger.error("Upload error: " + error.message); + logger.error(`Stack trace: ${error.stack}`); + if (tryRespondProductPersistenceError(error, res)) { + return; } - return res.status(500).send({ - status: "failed", - message: "Internal server error" + return sendUploadResponse(res, 500, { + status: "failed", + message: "Internal server error", }); } };