825 lines
24 KiB
JavaScript
825 lines
24 KiB
JavaScript
const fs = require("fs");
|
||
const csv = require("csv-parser");
|
||
const path = require("path");
|
||
const db = require("../models");
|
||
const { Sequelize } = require("sequelize");
|
||
const Product = db.Product;
|
||
const UnitMaster = db.UnitMaster;
|
||
const sanitize = require("sanitize-html");
|
||
const { UPLOAD_DIR } = require('../config/upload.config');
|
||
const logger = require("../services/logger");
|
||
|
||
const cleanString = (value) =>
|
||
typeof value === "string"
|
||
? sanitize(value, { allowedTags: [], allowedAttributes: {} })
|
||
: value;
|
||
|
||
/** Matches products.weight_in_ib DECIMAL(18,10): max 8 digits before the decimal. */
|
||
const WEIGHT_DECIMAL_PLACES = 10;
|
||
const WEIGHT_MAX_BEFORE_DECIMAL = 8;
|
||
const WEIGHT_MAX =
|
||
Number("9".repeat(WEIGHT_MAX_BEFORE_DECIMAL) + "." + "9".repeat(WEIGHT_DECIMAL_PLACES));
|
||
|
||
/** Parses optional products.weight_in_ib (nullable non-negative decimal). */
|
||
function parseWeightInIb(value) {
|
||
if (value === undefined || value === null || value === "") {
|
||
return { ok: true, value: null };
|
||
}
|
||
const str = String(value).trim().replace(",", ".");
|
||
const n = Number(str);
|
||
if (!Number.isFinite(n)) {
|
||
return { ok: false, message: "weight_in_ib must be a valid number." };
|
||
}
|
||
if (n < 0) {
|
||
return { ok: false, message: "weight_in_ib must be non-negative." };
|
||
}
|
||
const rounded = Number(n.toFixed(WEIGHT_DECIMAL_PLACES));
|
||
if (rounded > WEIGHT_MAX) {
|
||
return {
|
||
ok: false,
|
||
message: `weight_in_ib is too large. Maximum is ${WEIGHT_MAX} (${WEIGHT_MAX_BEFORE_DECIMAL} digits before the decimal).`,
|
||
};
|
||
}
|
||
return { ok: true, value: rounded };
|
||
}
|
||
|
||
/** Normalize unit text for case-insensitive matching (e.g. "numbers" → "numbers", " Numbers " → "numbers"). */
|
||
function normalizeUnitKey(value) {
|
||
if (value === null || value === undefined) return "";
|
||
const normalized = String(value).trim().replace(/\s+/g, " ").toLowerCase();
|
||
if (!normalized || normalized === "not defined") return "";
|
||
return normalized;
|
||
}
|
||
|
||
/** Map unit_master by uom (full unit name), case-insensitive. */
|
||
function buildUnitLookupMap(unitMasters) {
|
||
const unitMap = new Map();
|
||
for (const unit of unitMasters) {
|
||
const key = normalizeUnitKey(unit.uom);
|
||
if (key) {
|
||
unitMap.set(key, unit.id);
|
||
}
|
||
}
|
||
return unitMap;
|
||
}
|
||
|
||
function resolveUnitId(unitInput, unitMap) {
|
||
const key = normalizeUnitKey(unitInput);
|
||
if (!key) return null;
|
||
return unitMap.get(key) ?? null;
|
||
}
|
||
|
||
function normalizeCsvHeaderKey(key) {
|
||
if (!key) return "";
|
||
const withoutBom = String(key).replace(/^\uFEFF/, "");
|
||
return withoutBom
|
||
.replace(/\*/g, "")
|
||
.replace(/\(.*?\)/g, "")
|
||
.trim()
|
||
.replace(/[\s\W]+/g, "_")
|
||
.trim()
|
||
.toLowerCase();
|
||
}
|
||
|
||
function mapCsvRowKey(normalizedKey) {
|
||
if (/^hs.*code$/.test(normalizedKey) || normalizedKey === "hscode") {
|
||
return "hs_code";
|
||
}
|
||
if (/^product.*name$/.test(normalizedKey)) {
|
||
return "product_name";
|
||
}
|
||
if (
|
||
normalizedKey === "unit" ||
|
||
normalizedKey === "uom" ||
|
||
normalizedKey === "measurement_unit" ||
|
||
normalizedKey === "measure" ||
|
||
normalizedKey.endsWith("_unit")
|
||
) {
|
||
return "unit";
|
||
}
|
||
if (/^desc(ription)?$/.test(normalizedKey)) {
|
||
return "description";
|
||
}
|
||
if (
|
||
normalizedKey === "weight_in_ib" ||
|
||
normalizedKey === "weight_in_lb" ||
|
||
/^weight.*(ib|lb)/i.test(normalizedKey) ||
|
||
normalizedKey === "weight"
|
||
) {
|
||
return "weight_in_ib";
|
||
}
|
||
return normalizedKey;
|
||
}
|
||
|
||
function transformCsvRow(row) {
|
||
const cleanRow = {};
|
||
for (const key in row) {
|
||
if (!Object.prototype.hasOwnProperty.call(row, key)) continue;
|
||
const mappedKey = mapCsvRowKey(normalizeCsvHeaderKey(key));
|
||
const value =
|
||
row[key] !== undefined && row[key] !== null && String(row[key]).trim() !== ""
|
||
? String(row[key]).trim()
|
||
: null;
|
||
if (cleanRow[mappedKey] && !value) continue;
|
||
cleanRow[mappedKey] = value;
|
||
}
|
||
return cleanRow;
|
||
}
|
||
|
||
function isBlankCsvRow(row) {
|
||
return (
|
||
!row.hs_code &&
|
||
!row.product_name &&
|
||
!row.unit &&
|
||
!row.description &&
|
||
(row.weight_in_ib === undefined ||
|
||
row.weight_in_ib === null ||
|
||
String(row.weight_in_ib).trim() === "")
|
||
);
|
||
}
|
||
|
||
function parseCsvFile(filePath) {
|
||
return new Promise((resolve, reject) => {
|
||
const results = [];
|
||
fs.createReadStream(filePath)
|
||
.pipe(csv())
|
||
.on("data", (row) => results.push(transformCsvRow(row)))
|
||
.on("end", () => resolve(results))
|
||
.on("error", reject);
|
||
});
|
||
}
|
||
|
||
function sendUploadResponse(res, statusCode, body) {
|
||
if (res.headersSent) return;
|
||
return res.status(statusCode).json(body);
|
||
}
|
||
|
||
/** Returns true if a response was sent (4xx). */
|
||
function tryRespondProductPersistenceError(error, res) {
|
||
if (!error || res.headersSent) {
|
||
return false;
|
||
}
|
||
|
||
if (error.name === "SequelizeUniqueConstraintError") {
|
||
const field = error.errors?.[0]?.path ?? "field";
|
||
res.status(400).json({
|
||
status: "failed",
|
||
message: `${field} already exists.`,
|
||
});
|
||
return true;
|
||
}
|
||
|
||
const sqlMessage = error.parent?.sqlMessage || error.original?.sqlMessage || "";
|
||
const errno = error.parent?.errno ?? error.original?.errno;
|
||
const combined = `${error.message} ${sqlMessage}`;
|
||
|
||
if (error.name === "SequelizeDatabaseError") {
|
||
if (
|
||
combined.includes("weight_in_ib") ||
|
||
(errno === 1264 && /weight/i.test(combined))
|
||
) {
|
||
res.status(400).json({
|
||
status: "failed",
|
||
message:
|
||
"weight_in_ib is outside the range allowed by your database column (MySQL DECIMAL precision/scale). For example, DECIMAL(12,10) only allows values below 100. Either use a smaller weight, or widen the column (e.g. ALTER TABLE products MODIFY COLUMN weight_in_ib DECIMAL(18,10) NULL).",
|
||
});
|
||
return true;
|
||
}
|
||
}
|
||
|
||
return false;
|
||
}
|
||
|
||
exports.createProduct = async (req, res) => {
|
||
try {
|
||
const product_name = cleanString(req.body.product_name);
|
||
const hs_description = cleanString(req.body.hs_description);
|
||
const hs_code = req.body.hs_code;
|
||
const unit_id = req.body.unit_id;
|
||
|
||
const weightParsed = parseWeightInIb(req.body.weight_in_ib);
|
||
if (!weightParsed.ok) {
|
||
return res.status(400).json({ status: "failed", message: weightParsed.message });
|
||
}
|
||
|
||
if (hs_description && hs_description.length > 1000) {
|
||
return res.status(400).json({
|
||
status: "failed",
|
||
message: "HS Description is too long. Maximum allowed length is 1000 characters."
|
||
});
|
||
}
|
||
|
||
if (!/^\d+$/.test(hs_code)) {
|
||
return res.status(400).json({error: "Invalid HS Code: Must be numeric only."});
|
||
}
|
||
|
||
if (String(hs_code).length > 10 && String(hs_code).length < 1 && hs_code === 0 ) {
|
||
return res.status(400).send({status: "failed", message: "Invalid HS Code: maximum length is 10 digits."});
|
||
}
|
||
|
||
if(hs_code === 0 || String(hs_code) === "0"){
|
||
return res.status(400).send({status: "failed", message: " must be numeric, 1–10 digits, and cannot be 0."});
|
||
}
|
||
|
||
const productNameExists = await Product.findOne({
|
||
where: { product_name, is_active: true }
|
||
});
|
||
|
||
if (productNameExists) {
|
||
return res.status(400).json({
|
||
status: "failed",
|
||
message: `Product name '${product_name}' already exists.`
|
||
});
|
||
}
|
||
|
||
const hsCodeExists = await Product.findOne({
|
||
where: { hs_code, is_active: true }
|
||
});
|
||
|
||
if (hsCodeExists) {
|
||
return res.status(400).json({
|
||
status: "failed",
|
||
message: `HS Code '${hs_code}' already exists.`
|
||
});
|
||
}
|
||
|
||
await Product.create({
|
||
product_name,
|
||
hs_code,
|
||
hs_description,
|
||
unit_id,
|
||
weight_in_ib: weightParsed.value,
|
||
created_by: req.user.id
|
||
});
|
||
|
||
res.status(201).send({'status':"success",'message':"created successfully." });
|
||
} catch (error) {
|
||
logger.error(error.message);
|
||
logger.error(`Stack trace: ${error.stack}`);
|
||
if (tryRespondProductPersistenceError(error, res)) {
|
||
return;
|
||
}
|
||
res.status(500).send({'status':"failed",'message': "Internal server error" });
|
||
}
|
||
};
|
||
|
||
exports.getAllProducts = async (req, res) => {
|
||
try {
|
||
|
||
const data = await Product.findAll({
|
||
where: {
|
||
is_active: true
|
||
},
|
||
attributes: {
|
||
include: [
|
||
[
|
||
Sequelize.literal(
|
||
`(SELECT COUNT(*) FROM establishment_products AS EP WHERE EP.product_id = products.id)`
|
||
),
|
||
"mapped_establishment_count",
|
||
],
|
||
[
|
||
Sequelize.literal(`(SELECT name FROM admin_users au WHERE au.id = products.created_by)`),
|
||
"created_by_name"
|
||
]
|
||
],
|
||
},
|
||
include: [
|
||
{
|
||
model: UnitMaster,
|
||
as: "unit",
|
||
attributes: ["uom"],
|
||
},
|
||
],
|
||
order: [
|
||
[
|
||
Sequelize.literal(
|
||
"GREATEST(products.created_at, COALESCE(products.updated_at, products.created_at))"
|
||
),
|
||
"DESC",
|
||
],
|
||
["id", "DESC"],
|
||
],
|
||
});
|
||
|
||
res.status(200).send({
|
||
status: "success",
|
||
message: "Fetched successfully",
|
||
product_count: data.length,
|
||
data: data
|
||
});
|
||
|
||
} catch (error) {
|
||
logger.error(error.message);
|
||
logger.error(`Stack trace: ${error.stack}`);
|
||
res.status(500).send({ status: "failed", message: "Internal server error" });
|
||
}
|
||
};
|
||
|
||
exports.getProductById = async (req, res) => {
|
||
try {
|
||
|
||
const data = await Product.findByPk(req.params.id);
|
||
|
||
if (!data) res.status(404).send({'status':"failed",'message':"Record Not found",'data': "" });
|
||
|
||
res.status(200).send({'status':"success",'message':"Fetched successfully",'data': data });
|
||
|
||
} catch (error) {
|
||
logger.error(error.message);
|
||
logger.error(`Stack trace: ${error.stack}`);
|
||
res.status(500).send({'status':"failed",'message': "Internal server error" });
|
||
}
|
||
};
|
||
|
||
exports.updateProduct = async (req, res) => {
|
||
try {
|
||
|
||
const object = {
|
||
...req.body,
|
||
updated_by: req.user.id,
|
||
updated_at: new Date()
|
||
};
|
||
|
||
if (Object.prototype.hasOwnProperty.call(req.body, "weight_in_ib")) {
|
||
const weightParsed = parseWeightInIb(req.body.weight_in_ib);
|
||
if (!weightParsed.ok) {
|
||
return res.status(400).json({ status: "failed", message: weightParsed.message });
|
||
}
|
||
object.weight_in_ib = weightParsed.value;
|
||
}
|
||
|
||
const [updated] = await Product.update(object, { where: { id: req.params.id } });
|
||
|
||
if (!updated) res.status(404).send({'status':"failed",'message':"Record Not found",'data': "" });
|
||
|
||
res.status(200).send({'status':"success",'message':"Updated successfully",'data': "" });
|
||
|
||
} catch (error) {
|
||
logger.error(error.message);
|
||
logger.error(`Stack trace: ${error.stack}`);
|
||
if (tryRespondProductPersistenceError(error, res)) {
|
||
return;
|
||
}
|
||
res.status(500).send({'status':"failed",'message': "Internal server error" });
|
||
}
|
||
};
|
||
|
||
|
||
exports.deleteProduct = async (req, res) => {
|
||
try {
|
||
|
||
const deleted = await Product.destroy({ where: { id: req.params.id } });
|
||
|
||
if (!deleted) res.status(404).send({'status':"failed",'message':"Record Not found",'data': "" });
|
||
|
||
res.status(200).send({'status':"success",'message':"Deleted successfully",'data': "" });
|
||
|
||
} catch (error) {
|
||
logger.error(error.message);
|
||
logger.error(`Stack trace: ${error.stack}`);
|
||
res.status(500).send({'status':"failed",'message': "Internal server error" });
|
||
}
|
||
};
|
||
|
||
|
||
exports.downloadProductSample = async (req, res) => {
|
||
try {
|
||
const filePath = path.join(__dirname, "../downloads_csv/products_upload_sample.csv");
|
||
return res.download(filePath, "products_upload_sample.csv");
|
||
} catch (error) {
|
||
logger.error(error.message);
|
||
logger.error(`Stack trace: ${error.stack}`);
|
||
res.status(500).send({'status':"failed",'message': "Internal server error" });
|
||
}
|
||
};
|
||
|
||
function sanitizeFilePath(userInput, allowedDirectory) {
|
||
if (!userInput || typeof userInput !== 'string') {
|
||
throw new Error('Invalid file path input');
|
||
}
|
||
|
||
let sanitized = userInput.replace(/\.\./g, '');
|
||
sanitized = sanitized.replace(/[\/\\]+/g, path.sep);
|
||
const filename = path.basename(sanitized);
|
||
const fullPath = path.join(allowedDirectory, filename);
|
||
const resolvedPath = path.resolve(fullPath);
|
||
const resolvedBase = path.resolve(allowedDirectory);
|
||
|
||
if (!resolvedPath.startsWith(resolvedBase)) {
|
||
throw new Error('Path traversal attempt detected');
|
||
}
|
||
return resolvedPath;
|
||
}
|
||
|
||
function validateFileExists(filePath) {
|
||
if (!filePath) {
|
||
return false;
|
||
}
|
||
|
||
try {
|
||
return fs.existsSync(filePath);
|
||
} catch (err) {
|
||
return false;
|
||
}
|
||
}
|
||
|
||
function deleteFileSecure(filePath) {
|
||
if (!filePath) {
|
||
return;
|
||
}
|
||
|
||
try {
|
||
if (validateFileExists(filePath)) {
|
||
fs.unlinkSync(filePath);
|
||
}
|
||
} catch (err) {
|
||
if (logger && logger.error) {
|
||
logger.error('File deletion error: ' + err.message);
|
||
logger.error(`Stack trace: ${err.stack}`);
|
||
}
|
||
}
|
||
}
|
||
|
||
exports.uploadProductsFromCSV = async (req, res) => {
|
||
let sanitizedPath = null;
|
||
|
||
try {
|
||
if (!req.file) {
|
||
return sendUploadResponse(res, 400, {
|
||
status: "failed",
|
||
message:
|
||
'No CSV file attached. Use multipart form field "file" and select your .csv before submitting.',
|
||
hint:
|
||
'In Swagger: open Try it out → pick hs-codes-sample.csv under field "file" → Execute.',
|
||
expected_field: "file",
|
||
});
|
||
}
|
||
|
||
try {
|
||
sanitizedPath = sanitizeFilePath(req.file.path, UPLOAD_DIR);
|
||
} catch (sanitizeError) {
|
||
try {
|
||
const unsafePath = req.file.path;
|
||
if (unsafePath && fs.existsSync(unsafePath)) {
|
||
fs.unlinkSync(unsafePath);
|
||
}
|
||
} catch (cleanupErr) {
|
||
// ignore cleanup errors
|
||
}
|
||
return sendUploadResponse(res, 400, {
|
||
status: "failed",
|
||
message: "Invalid file path detected",
|
||
});
|
||
}
|
||
|
||
if (!validateFileExists(sanitizedPath)) {
|
||
return sendUploadResponse(res, 400, {
|
||
status: "failed",
|
||
message: "File not found after validation",
|
||
});
|
||
}
|
||
|
||
const originalName = path.basename(req.file.originalname);
|
||
const fileExt = path.extname(originalName).toLowerCase();
|
||
|
||
if (fileExt !== ".csv") {
|
||
deleteFileSecure(sanitizedPath);
|
||
return sendUploadResponse(res, 400, {
|
||
status: "failed",
|
||
message: "Invalid file type. Only CSV allowed.",
|
||
});
|
||
}
|
||
|
||
const allowedMimes = ["text/csv", "application/csv", "text/plain", "application/vnd.ms-excel"];
|
||
if (req.file.mimetype && !allowedMimes.includes(req.file.mimetype)) {
|
||
deleteFileSecure(sanitizedPath);
|
||
return sendUploadResponse(res, 400, {
|
||
status: "failed",
|
||
message: "Invalid MIME type. Only CSV allowed.",
|
||
});
|
||
}
|
||
|
||
if (!req.user || !req.user.id || Number.isNaN(Number(req.user.id))) {
|
||
deleteFileSecure(sanitizedPath);
|
||
return sendUploadResponse(res, 400, {
|
||
status: "failed",
|
||
message: "Invalid or missing User Id.",
|
||
});
|
||
}
|
||
|
||
const fileStats = fs.statSync(sanitizedPath);
|
||
if (fileStats.size === 0) {
|
||
deleteFileSecure(sanitizedPath);
|
||
return sendUploadResponse(res, 400, {
|
||
status: "failed",
|
||
message: "Uploaded file is empty.",
|
||
});
|
||
}
|
||
|
||
const userId = parseInt(req.user.id, 10);
|
||
const csvResults = await parseCsvFile(sanitizedPath);
|
||
|
||
if (csvResults.length === 0) {
|
||
deleteFileSecure(sanitizedPath);
|
||
return sendUploadResponse(res, 400, {
|
||
status: "failed",
|
||
message: "CSV file is empty or invalid.",
|
||
});
|
||
}
|
||
|
||
const requiredCols = ["hs_code", "product_name", "unit"];
|
||
const optionalCols = ["description", "weight_in_ib"];
|
||
const allowedCols = requiredCols.concat(optionalCols);
|
||
const headers = Object.keys(csvResults[0]);
|
||
|
||
const missingCols = requiredCols.filter((col) => !headers.includes(col));
|
||
const extraCols = headers.filter((col) => !allowedCols.includes(col));
|
||
|
||
if (missingCols.length > 0 || extraCols.length > 0) {
|
||
deleteFileSecure(sanitizedPath);
|
||
return sendUploadResponse(res, 400, {
|
||
status: "failed",
|
||
message:
|
||
(missingCols.length ? `Missing columns: ${missingCols.join(", ")}. ` : "") +
|
||
(extraCols.length ? `Unexpected columns: ${extraCols.join(", ")}.` : ""),
|
||
});
|
||
}
|
||
|
||
const normalizedRows = [];
|
||
const seenHsCodes = new Set();
|
||
const fileDuplicates = new Set();
|
||
const validationErrors = [];
|
||
|
||
for (let index = 0; index < csvResults.length; index++) {
|
||
const row = csvResults[index];
|
||
const csvLineNumber = index + 2;
|
||
|
||
if (isBlankCsvRow(row)) {
|
||
continue;
|
||
}
|
||
|
||
const hsCode = row.hs_code ? row.hs_code.replace(/[-\s/]/g, "").trim() : "";
|
||
const productName = row.product_name
|
||
? row.product_name.replace(/\s+/g, " ").trim()
|
||
: "";
|
||
const unit = normalizeUnitKey(row.unit);
|
||
const description = row.description ? row.description.trim() : "";
|
||
|
||
let weightInIb = null;
|
||
if (
|
||
row.weight_in_ib !== undefined &&
|
||
row.weight_in_ib !== null &&
|
||
String(row.weight_in_ib).trim() !== ""
|
||
) {
|
||
const wp = parseWeightInIb(row.weight_in_ib);
|
||
if (!wp.ok) {
|
||
validationErrors.push({ row: csvLineNumber, error: wp.message });
|
||
continue;
|
||
}
|
||
weightInIb = wp.value;
|
||
}
|
||
|
||
if (!hsCode || !productName || !unit) {
|
||
const missing = [];
|
||
if (!hsCode) missing.push("HS Code");
|
||
if (!productName) missing.push("Product Name");
|
||
if (!unit) missing.push("Unit");
|
||
validationErrors.push({
|
||
row: csvLineNumber,
|
||
error: `Missing required field(s): ${missing.join(", ")}`,
|
||
});
|
||
continue;
|
||
}
|
||
|
||
if (!/^\d+$/.test(hsCode)) {
|
||
validationErrors.push({ row: csvLineNumber, error: "HS Code must be numeric" });
|
||
continue;
|
||
}
|
||
|
||
if (hsCode.length > 10) {
|
||
validationErrors.push({
|
||
row: csvLineNumber,
|
||
error: "HS Code must be max 10 digits",
|
||
});
|
||
continue;
|
||
}
|
||
|
||
if (hsCode === "0" || /^0+$/.test(hsCode)) {
|
||
validationErrors.push({
|
||
row: csvLineNumber,
|
||
error: "HS Code cannot be 0",
|
||
});
|
||
continue;
|
||
}
|
||
|
||
if (productName.length > 1000) {
|
||
validationErrors.push({
|
||
row: csvLineNumber,
|
||
error: "Product Name is too long. Maximum 1000 characters.",
|
||
});
|
||
continue;
|
||
}
|
||
|
||
if (description.length > 1000) {
|
||
validationErrors.push({
|
||
row: csvLineNumber,
|
||
error: "HS Description is too long. Maximum 1000 characters.",
|
||
});
|
||
continue;
|
||
}
|
||
|
||
const normalizedHs = hsCode.replace(/^0+/, "") || hsCode;
|
||
if (seenHsCodes.has(normalizedHs)) {
|
||
fileDuplicates.add(hsCode);
|
||
continue;
|
||
}
|
||
seenHsCodes.add(normalizedHs);
|
||
|
||
normalizedRows.push({
|
||
hsCode,
|
||
productName,
|
||
unit,
|
||
description,
|
||
weightInIb,
|
||
csvLineNumber,
|
||
});
|
||
}
|
||
|
||
if (normalizedRows.length === 0 && validationErrors.length === 0) {
|
||
deleteFileSecure(sanitizedPath);
|
||
return sendUploadResponse(res, 400, {
|
||
status: "failed",
|
||
message:
|
||
"No product data rows found. Add data below the header row (remove blank rows from the template).",
|
||
});
|
||
}
|
||
|
||
if (fileDuplicates.size > 0) {
|
||
deleteFileSecure(sanitizedPath);
|
||
return sendUploadResponse(res, 400, {
|
||
status: "failed",
|
||
message:
|
||
"Duplicate HS Codes found within file. Resolve and re-upload.",
|
||
duplicate_hs_codes_in_file: Array.from(fileDuplicates),
|
||
});
|
||
}
|
||
|
||
const unitMasters = await UnitMaster.findAll({
|
||
attributes: ["id", "uom"],
|
||
where: { is_active: true },
|
||
});
|
||
const unitMap = buildUnitLookupMap(unitMasters);
|
||
|
||
const existingProducts = await Product.findAll({
|
||
attributes: ["hs_code", "product_name"],
|
||
where: { is_active: true },
|
||
});
|
||
|
||
const existingHsCodeSet = new Set();
|
||
const existingProductNameSet = new Set();
|
||
for (const product of existingProducts) {
|
||
if (product.hs_code != null && String(product.hs_code).trim() !== "") {
|
||
existingHsCodeSet.add(String(product.hs_code).replace(/^0+/, "") || "0");
|
||
}
|
||
if (product.product_name) {
|
||
existingProductNameSet.add(product.product_name.trim().toLowerCase());
|
||
}
|
||
}
|
||
|
||
const duplicateHsCodes = [];
|
||
const duplicateProductNames = [];
|
||
|
||
for (let i = 0; i < normalizedRows.length; i++) {
|
||
const row = normalizedRows[i];
|
||
const normalizedHs = row.hsCode.replace(/^0+/, "") || row.hsCode;
|
||
const productNameLower = row.productName.trim().toLowerCase();
|
||
|
||
if (existingHsCodeSet.has(normalizedHs)) {
|
||
duplicateHsCodes.push({
|
||
row: row.csvLineNumber,
|
||
hs_code: row.hsCode,
|
||
product_name: row.productName,
|
||
});
|
||
}
|
||
|
||
if (existingProductNameSet.has(productNameLower)) {
|
||
duplicateProductNames.push({
|
||
row: row.csvLineNumber,
|
||
hs_code: row.hsCode,
|
||
product_name: row.productName,
|
||
});
|
||
}
|
||
}
|
||
|
||
if (duplicateHsCodes.length > 0 || duplicateProductNames.length > 0) {
|
||
deleteFileSecure(sanitizedPath);
|
||
const errorMessages = [];
|
||
if (duplicateHsCodes.length > 0) {
|
||
errorMessages.push(
|
||
`${duplicateHsCodes.length} duplicate HS Code(s) found in database`
|
||
);
|
||
}
|
||
if (duplicateProductNames.length > 0) {
|
||
errorMessages.push(
|
||
`${duplicateProductNames.length} duplicate Product Name(s) found in database`
|
||
);
|
||
}
|
||
return sendUploadResponse(res, 400, {
|
||
status: "failed",
|
||
message:
|
||
"Upload rejected: " +
|
||
errorMessages.join(", ") +
|
||
". Please remove duplicates and try again.",
|
||
duplicate_hs_codes: duplicateHsCodes,
|
||
duplicate_product_names: duplicateProductNames,
|
||
total_duplicates:
|
||
duplicateHsCodes.length + duplicateProductNames.length,
|
||
});
|
||
}
|
||
|
||
const toInsert = [];
|
||
|
||
for (let i = 0; i < normalizedRows.length; i++) {
|
||
const row = normalizedRows[i];
|
||
const unitId = resolveUnitId(row.unit, unitMap);
|
||
|
||
if (!unitId) {
|
||
validationErrors.push({
|
||
row: row.csvLineNumber,
|
||
error: `Invalid unit "${row.unit}". Use the unit name (uom) from Unit Master (e.g. Numbers, Kilogram). Matching is case-insensitive.`,
|
||
});
|
||
continue;
|
||
}
|
||
|
||
toInsert.push({
|
||
hs_code: parseInt(row.hsCode, 10),
|
||
product_name: row.productName,
|
||
unit_id: unitId,
|
||
hs_description: row.description || null,
|
||
weight_in_ib: row.weightInIb,
|
||
created_by: userId,
|
||
created_at: new Date(),
|
||
is_active: true,
|
||
});
|
||
}
|
||
|
||
if (validationErrors.length > 0) {
|
||
deleteFileSecure(sanitizedPath);
|
||
return sendUploadResponse(res, 400, {
|
||
status: "failed",
|
||
message: `Upload rejected: ${validationErrors.length} validation error(s) found. Please fix and try again.`,
|
||
errors: validationErrors,
|
||
});
|
||
}
|
||
|
||
if (toInsert.length === 0) {
|
||
deleteFileSecure(sanitizedPath);
|
||
return sendUploadResponse(res, 400, {
|
||
status: "failed",
|
||
message: "No valid product rows to import.",
|
||
});
|
||
}
|
||
|
||
let insertedRecords = [];
|
||
try {
|
||
insertedRecords = await Product.bulkCreate(toInsert, {
|
||
validate: true,
|
||
individualHooks: false,
|
||
});
|
||
} catch (bulkError) {
|
||
deleteFileSecure(sanitizedPath);
|
||
logger.error("Product bulkCreate error: " + bulkError.message);
|
||
logger.error(`Stack trace: ${bulkError.stack}`);
|
||
if (tryRespondProductPersistenceError(bulkError, res)) {
|
||
return;
|
||
}
|
||
throw bulkError;
|
||
}
|
||
|
||
deleteFileSecure(sanitizedPath);
|
||
|
||
return sendUploadResponse(res, 201, {
|
||
status: "success",
|
||
message: `${insertedRecords.length} product(s) inserted successfully.`,
|
||
summary: {
|
||
total_records: csvResults.length,
|
||
imported: insertedRecords.length,
|
||
skipped: csvResults.length - insertedRecords.length,
|
||
errors: [],
|
||
},
|
||
});
|
||
} catch (error) {
|
||
deleteFileSecure(sanitizedPath);
|
||
logger.error("Upload error: " + error.message);
|
||
logger.error(`Stack trace: ${error.stack}`);
|
||
if (tryRespondProductPersistenceError(error, res)) {
|
||
return;
|
||
}
|
||
return sendUploadResponse(res, 500, {
|
||
status: "failed",
|
||
message: "Internal server error",
|
||
});
|
||
}
|
||
};
|