GWM : product upload issue
This commit is contained in:
parent
35fb4dacf1
commit
ca936f8f7e
@ -43,6 +43,105 @@ function parseWeightInIb(value) {
|
||||
return { ok: true, value: rounded };
|
||||
}
|
||||
|
||||
/** Map unit_master by full name (uom) and short code (uom_short_name), case-insensitive. */
|
||||
function buildUnitLookupMap(unitMasters) {
|
||||
const unitMap = new Map();
|
||||
for (const unit of unitMasters) {
|
||||
if (unit.uom) {
|
||||
unitMap.set(unit.uom.trim().toLowerCase(), unit.id);
|
||||
}
|
||||
if (unit.uom_short_name) {
|
||||
unitMap.set(unit.uom_short_name.trim().toLowerCase(), unit.id);
|
||||
}
|
||||
}
|
||||
return unitMap;
|
||||
}
|
||||
|
||||
function normalizeCsvHeaderKey(key) {
|
||||
if (!key) return "";
|
||||
const withoutBom = String(key).replace(/^\uFEFF/, "");
|
||||
return withoutBom
|
||||
.replace(/\*/g, "")
|
||||
.replace(/\(.*?\)/g, "")
|
||||
.trim()
|
||||
.replace(/[\s\W]+/g, "_")
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
}
|
||||
|
||||
function mapCsvRowKey(normalizedKey) {
|
||||
if (/^hs.*code$/.test(normalizedKey) || normalizedKey === "hscode") {
|
||||
return "hs_code";
|
||||
}
|
||||
if (/^product.*name$/.test(normalizedKey)) {
|
||||
return "product_name";
|
||||
}
|
||||
if (
|
||||
normalizedKey === "unit" ||
|
||||
normalizedKey === "uom" ||
|
||||
normalizedKey === "measurement_unit" ||
|
||||
normalizedKey === "measure" ||
|
||||
normalizedKey.endsWith("_unit")
|
||||
) {
|
||||
return "unit";
|
||||
}
|
||||
if (/^desc(ription)?$/.test(normalizedKey)) {
|
||||
return "description";
|
||||
}
|
||||
if (
|
||||
normalizedKey === "weight_in_ib" ||
|
||||
normalizedKey === "weight_in_lb" ||
|
||||
/^weight.*(ib|lb)/i.test(normalizedKey) ||
|
||||
normalizedKey === "weight"
|
||||
) {
|
||||
return "weight_in_ib";
|
||||
}
|
||||
return normalizedKey;
|
||||
}
|
||||
|
||||
function transformCsvRow(row) {
|
||||
const cleanRow = {};
|
||||
for (const key in row) {
|
||||
if (!Object.prototype.hasOwnProperty.call(row, key)) continue;
|
||||
const mappedKey = mapCsvRowKey(normalizeCsvHeaderKey(key));
|
||||
const value =
|
||||
row[key] !== undefined && row[key] !== null && String(row[key]).trim() !== ""
|
||||
? String(row[key]).trim()
|
||||
: null;
|
||||
if (cleanRow[mappedKey] && !value) continue;
|
||||
cleanRow[mappedKey] = value;
|
||||
}
|
||||
return cleanRow;
|
||||
}
|
||||
|
||||
function isBlankCsvRow(row) {
|
||||
return (
|
||||
!row.hs_code &&
|
||||
!row.product_name &&
|
||||
!row.unit &&
|
||||
!row.description &&
|
||||
(row.weight_in_ib === undefined ||
|
||||
row.weight_in_ib === null ||
|
||||
String(row.weight_in_ib).trim() === "")
|
||||
);
|
||||
}
|
||||
|
||||
function parseCsvFile(filePath) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const results = [];
|
||||
fs.createReadStream(filePath)
|
||||
.pipe(csv())
|
||||
.on("data", (row) => results.push(transformCsvRow(row)))
|
||||
.on("end", () => resolve(results))
|
||||
.on("error", reject);
|
||||
});
|
||||
}
|
||||
|
||||
function sendUploadResponse(res, statusCode, body) {
|
||||
if (res.headersSent) return;
|
||||
return res.status(statusCode).json(body);
|
||||
}
|
||||
|
||||
/** Returns true if a response was sent (4xx). */
|
||||
function tryRespondProductPersistenceError(error, res) {
|
||||
if (!error || res.headersSent) {
|
||||
@ -334,420 +433,376 @@ exports.uploadProductsFromCSV = async (req, res) => {
|
||||
let sanitizedPath = null;
|
||||
|
||||
try {
|
||||
// Validate file upload
|
||||
if (!req.file) {
|
||||
return res.status(400).send({
|
||||
status: "failed",
|
||||
message: "No file uploaded"
|
||||
return sendUploadResponse(res, 400, {
|
||||
status: "failed",
|
||||
message: "No file uploaded",
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
try {
|
||||
sanitizedPath = sanitizeFilePath(req.file.path, UPLOAD_DIR);
|
||||
} catch (sanitizeError) {
|
||||
// Attempt cleanup with original path if sanitization fails
|
||||
try {
|
||||
const unsafePath = req.file.path;
|
||||
if (unsafePath && fs.existsSync(unsafePath)) {
|
||||
fs.unlinkSync(unsafePath);
|
||||
}
|
||||
} catch (cleanupErr) {
|
||||
// Silent fail on cleanup
|
||||
// ignore cleanup errors
|
||||
}
|
||||
|
||||
return res.status(400).send({
|
||||
return sendUploadResponse(res, 400, {
|
||||
status: "failed",
|
||||
message: "Invalid file path detected"
|
||||
message: "Invalid file path detected",
|
||||
});
|
||||
}
|
||||
|
||||
// Validate file exists after sanitization
|
||||
if (!validateFileExists(sanitizedPath)) {
|
||||
return res.status(400).send({
|
||||
return sendUploadResponse(res, 400, {
|
||||
status: "failed",
|
||||
message: "File not found after validation"
|
||||
message: "File not found after validation",
|
||||
});
|
||||
}
|
||||
|
||||
const originalName = path.basename(req.file.originalname);
|
||||
const fileExt = path.extname(originalName).toLowerCase();
|
||||
|
||||
if (fileExt !== '.csv') {
|
||||
deleteFileSecure(sanitizedPath);
|
||||
return res.status(400).send({
|
||||
status: "failed",
|
||||
message: "Invalid file type. Only CSV allowed."
|
||||
});
|
||||
}
|
||||
|
||||
// Validate MIME type
|
||||
const allowedMimes = ['text/csv', 'application/csv', 'text/plain'];
|
||||
if (req.file.mimetype && !allowedMimes.some(mime => mime === req.file.mimetype)) {
|
||||
if (fileExt !== ".csv") {
|
||||
deleteFileSecure(sanitizedPath);
|
||||
return res.status(400).send({
|
||||
return sendUploadResponse(res, 400, {
|
||||
status: "failed",
|
||||
message: "Invalid MIME type. Only CSV allowed."
|
||||
message: "Invalid file type. Only CSV allowed.",
|
||||
});
|
||||
}
|
||||
|
||||
// Validate user ID
|
||||
if (!req.user || !req.user.id || isNaN(req.user.id)) {
|
||||
const allowedMimes = ["text/csv", "application/csv", "text/plain", "application/vnd.ms-excel"];
|
||||
if (req.file.mimetype && !allowedMimes.includes(req.file.mimetype)) {
|
||||
deleteFileSecure(sanitizedPath);
|
||||
return res.status(400).send({
|
||||
status: "failed",
|
||||
message: "Invalid or missing User Id."
|
||||
return sendUploadResponse(res, 400, {
|
||||
status: "failed",
|
||||
message: "Invalid MIME type. Only CSV allowed.",
|
||||
});
|
||||
}
|
||||
|
||||
if (!req.user || !req.user.id || Number.isNaN(Number(req.user.id))) {
|
||||
deleteFileSecure(sanitizedPath);
|
||||
return sendUploadResponse(res, 400, {
|
||||
status: "failed",
|
||||
message: "Invalid or missing User Id.",
|
||||
});
|
||||
}
|
||||
|
||||
// Check file size
|
||||
const fileStats = fs.statSync(sanitizedPath);
|
||||
if (fileStats.size === 0) {
|
||||
deleteFileSecure(sanitizedPath);
|
||||
return res.status(400).send({
|
||||
return sendUploadResponse(res, 400, {
|
||||
status: "failed",
|
||||
message: "Uploaded file is empty.",
|
||||
});
|
||||
}
|
||||
|
||||
const csvResults = [];
|
||||
const userId = parseInt(req.user.id, 10);
|
||||
const csvResults = await parseCsvFile(sanitizedPath);
|
||||
|
||||
// Process CSV using sanitized path
|
||||
fs.createReadStream(sanitizedPath)
|
||||
.pipe(csv())
|
||||
.on("data", (row) => {
|
||||
const cleanRow = {};
|
||||
|
||||
for (const key in row) {
|
||||
if (!row.hasOwnProperty(key)) continue;
|
||||
|
||||
let cleanedHeader = key
|
||||
.replace(/\*/g, "")
|
||||
.replace(/\(.*?\)/g, "")
|
||||
.trim();
|
||||
|
||||
const normalizedKey = cleanedHeader
|
||||
.replace(/[\s\W]+/g, "_")
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
|
||||
let mappedKey = normalizedKey;
|
||||
if (/^hs.*code$/.test(normalizedKey) || normalizedKey === "hscode") {
|
||||
mappedKey = "hs_code";
|
||||
} else if (/^product.*name$/.test(normalizedKey)) {
|
||||
mappedKey = "product_name";
|
||||
} else if (/unit|measurement|uom|measure/i.test(normalizedKey)) {
|
||||
mappedKey = "unit";
|
||||
} else if (/desc(ription)?/i.test(normalizedKey)) {
|
||||
mappedKey = "description";
|
||||
} else if (
|
||||
normalizedKey === "weight_in_ib" ||
|
||||
/^weight.*(ib|lb)/i.test(normalizedKey) ||
|
||||
/^weight$/i.test(normalizedKey)
|
||||
) {
|
||||
mappedKey = "weight_in_ib";
|
||||
}
|
||||
|
||||
cleanRow[mappedKey] = row[key] ? row[key].trim() : null;
|
||||
}
|
||||
|
||||
csvResults.push(cleanRow);
|
||||
})
|
||||
.on("end", async () => {
|
||||
try {
|
||||
// Validate CSV has data
|
||||
if (csvResults.length === 0) {
|
||||
deleteFileSecure(sanitizedPath);
|
||||
return res.status(400).send({
|
||||
status: "failed",
|
||||
message: "CSV file is empty or invalid."
|
||||
});
|
||||
}
|
||||
|
||||
// Validate required columns
|
||||
const requiredCols = ["hs_code", "product_name", "unit", "description"];
|
||||
const optionalCols = ["weight_in_ib"];
|
||||
const allowedCols = requiredCols.concat(optionalCols);
|
||||
const headers = Object.keys(csvResults[0]);
|
||||
|
||||
const missingCols = requiredCols.filter(col => !headers.includes(col));
|
||||
const extraCols = headers.filter(col => !allowedCols.includes(col));
|
||||
|
||||
if (missingCols.length > 0 || extraCols.length > 0) {
|
||||
deleteFileSecure(sanitizedPath);
|
||||
return res.status(400).send({
|
||||
status: "failed",
|
||||
message:
|
||||
(missingCols.length ? `Missing columns: ${missingCols.join(", ")}. ` : "") +
|
||||
(extraCols.length ? `Unexpected columns: ${extraCols.join(", ")}.` : "")
|
||||
});
|
||||
}
|
||||
|
||||
// Process and validate rows
|
||||
const normalizedRows = [];
|
||||
const seenHsCodes = new Set();
|
||||
const fileDuplicates = new Set();
|
||||
const validationErrors = [];
|
||||
|
||||
for (let index = 0; index < csvResults.length; index++) {
|
||||
const row = csvResults[index];
|
||||
const rowNumber = index + 1;
|
||||
|
||||
let hsCode = row.hs_code ? row.hs_code.replace(/[-\s/]/g, "").trim() : "";
|
||||
const productName = row.product_name ? row.product_name.replace(/\s+/g, " ").trim() : "";
|
||||
const unit = row.unit ? row.unit.trim().toLowerCase() : "";
|
||||
const description = row.description ? row.description.trim().toLowerCase() : "";
|
||||
|
||||
let weightInIb = null;
|
||||
if (
|
||||
row.weight_in_ib !== undefined &&
|
||||
row.weight_in_ib !== null &&
|
||||
String(row.weight_in_ib).trim() !== ""
|
||||
) {
|
||||
const wp = parseWeightInIb(row.weight_in_ib);
|
||||
if (!wp.ok) {
|
||||
validationErrors.push({
|
||||
row: rowNumber,
|
||||
error: wp.message,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
weightInIb = wp.value;
|
||||
}
|
||||
|
||||
// Validate required fields
|
||||
if (!hsCode || !productName) {
|
||||
validationErrors.push({
|
||||
row: rowNumber,
|
||||
error: "Missing required HS Code or Product Name"
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
// Validate HS code format
|
||||
if (!/^\d+$/.test(hsCode)) {
|
||||
validationErrors.push({
|
||||
row: rowNumber,
|
||||
error: "HS Code must be numeric"
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
// Validate HS code length
|
||||
if (hsCode.length > 10) {
|
||||
validationErrors.push({
|
||||
row: rowNumber,
|
||||
error: "HS Code must be max 10 digits"
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
// Validate product name length
|
||||
if (productName.length > 1000) {
|
||||
validationErrors.push({
|
||||
row: rowNumber,
|
||||
error: "Product Name is too long. Maximum 1000 characters."
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
// Validate description length
|
||||
if (description.length > 1000) {
|
||||
validationErrors.push({
|
||||
row: rowNumber,
|
||||
error: "HS Description is too long. Maximum 1000 characters."
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check for duplicates in file
|
||||
const normalizedHs = hsCode.replace(/^0+/, "");
|
||||
if (seenHsCodes.has(normalizedHs)) {
|
||||
fileDuplicates.add(hsCode);
|
||||
continue;
|
||||
}
|
||||
seenHsCodes.add(normalizedHs);
|
||||
|
||||
normalizedRows.push({
|
||||
hsCode: hsCode,
|
||||
productName: productName,
|
||||
unit: unit,
|
||||
description: description,
|
||||
weightInIb,
|
||||
});
|
||||
}
|
||||
|
||||
// Check for file duplicates
|
||||
if (fileDuplicates.size > 0) {
|
||||
deleteFileSecure(sanitizedPath);
|
||||
return res.status(400).send({
|
||||
status: "failed",
|
||||
message: "Duplicate HS Codes found within file. Resolve and re-upload.",
|
||||
duplicate_hs_codes_in_file: Array.from(fileDuplicates)
|
||||
});
|
||||
}
|
||||
|
||||
const unitMasters = await UnitMaster.findAll({
|
||||
attributes: ["id", "uom"]
|
||||
});
|
||||
|
||||
const unitMap = new Map();
|
||||
for (let i = 0; i < unitMasters.length; i++) {
|
||||
const unit = unitMasters[i];
|
||||
const key = unit.uom.trim().toLowerCase();
|
||||
unitMap.set(key, unit.id);
|
||||
}
|
||||
|
||||
const existingProducts = await Product.findAll({
|
||||
attributes: ["hs_code", "product_name"]
|
||||
});
|
||||
|
||||
const existingHsCodeSet = new Set();
|
||||
const existingProductNameSet = new Set();
|
||||
for (let i = 0; i < existingProducts.length; i++) {
|
||||
const normalized = existingProducts[i].hs_code.replace(/^0+/, "");
|
||||
existingHsCodeSet.add(normalized);
|
||||
|
||||
const productNameLower = existingProducts[i].product_name.trim().toLowerCase();
|
||||
existingProductNameSet.add(productNameLower);
|
||||
}
|
||||
|
||||
const duplicateHsCodes = [];
|
||||
const duplicateProductNames = [];
|
||||
|
||||
for (let i = 0; i < normalizedRows.length; i++) {
|
||||
const row = normalizedRows[i];
|
||||
const normalizedHs = row.hsCode.replace(/^0+/, "");
|
||||
const productNameLower = row.productName.trim().toLowerCase();
|
||||
|
||||
if (existingHsCodeSet.has(normalizedHs)) {
|
||||
duplicateHsCodes.push({
|
||||
row: i + 1,
|
||||
hs_code: row.hsCode,
|
||||
product_name: row.productName
|
||||
});
|
||||
}
|
||||
|
||||
if (existingProductNameSet.has(productNameLower)) {
|
||||
duplicateProductNames.push({
|
||||
row: i + 1,
|
||||
hs_code: row.hsCode,
|
||||
product_name: row.productName
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (duplicateHsCodes.length > 0 || duplicateProductNames.length > 0) {
|
||||
deleteFileSecure(sanitizedPath);
|
||||
|
||||
const errorMessages = [];
|
||||
|
||||
if (duplicateHsCodes.length > 0) {
|
||||
errorMessages.push(`${duplicateHsCodes.length} duplicate HS Code(s) found in database`);
|
||||
}
|
||||
|
||||
if (duplicateProductNames.length > 0) {
|
||||
errorMessages.push(`${duplicateProductNames.length} duplicate Product Name(s) found in database`);
|
||||
}
|
||||
|
||||
return res.status(400).send({
|
||||
status: "failed",
|
||||
message: "Upload rejected: " + errorMessages.join(", ") + ". Please remove duplicates and try again.",
|
||||
duplicate_hs_codes: duplicateHsCodes,
|
||||
duplicate_product_names: duplicateProductNames,
|
||||
total_duplicates: duplicateHsCodes.length + duplicateProductNames.length
|
||||
});
|
||||
}
|
||||
|
||||
const toInsert = [];
|
||||
|
||||
for (let i = 0; i < normalizedRows.length; i++) {
|
||||
const row = normalizedRows[i];
|
||||
|
||||
// Validate unit
|
||||
const unitId = unitMap.get(row.unit);
|
||||
if (!unitId) {
|
||||
validationErrors.push({
|
||||
row: i + 1,
|
||||
error: "Invalid unit: " + row.unit
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
toInsert.push({
|
||||
hs_code: row.hsCode,
|
||||
product_name: row.productName,
|
||||
unit_id: unitId,
|
||||
hs_description: row.description,
|
||||
weight_in_ib: row.weightInIb,
|
||||
created_by: userId,
|
||||
created_at: new Date(),
|
||||
});
|
||||
}
|
||||
|
||||
if (validationErrors.length > 0) {
|
||||
deleteFileSecure(sanitizedPath);
|
||||
return res.status(400).send({
|
||||
status: "failed",
|
||||
message: `Upload rejected: ${validationErrors.length} validation error(s) found. Please fix and try again.`,
|
||||
errors: validationErrors
|
||||
});
|
||||
}
|
||||
|
||||
let insertedRecords = [];
|
||||
if (toInsert.length > 0) {
|
||||
insertedRecords = await Product.bulkCreate(toInsert, {
|
||||
validate: true
|
||||
});
|
||||
}
|
||||
|
||||
deleteFileSecure(sanitizedPath);
|
||||
|
||||
return res.status(200).send({
|
||||
status: "success",
|
||||
message: `${insertedRecords.length} products inserted successfully.`,
|
||||
summary: {
|
||||
total_records: csvResults.length,
|
||||
imported: insertedRecords.length,
|
||||
skipped: 0,
|
||||
errors: []
|
||||
}
|
||||
});
|
||||
|
||||
} catch (processingError) {
|
||||
deleteFileSecure(sanitizedPath);
|
||||
if (logger && logger.error) {
|
||||
logger.error("CSV processing error: " + processingError.message);
|
||||
logger.error(`Stack trace: ${processingError.stack}`);
|
||||
}
|
||||
if (tryRespondProductPersistenceError(processingError, res)) {
|
||||
return;
|
||||
}
|
||||
return res.status(500).send({
|
||||
status: "failed",
|
||||
message: "Internal server error",
|
||||
});
|
||||
}
|
||||
})
|
||||
.on("error", (streamError) => {
|
||||
deleteFileSecure(sanitizedPath);
|
||||
if (logger && logger.error) {
|
||||
logger.error("Stream error: " + streamError.message);
|
||||
}
|
||||
return res.status(500).send({
|
||||
status: "failed",
|
||||
message: "Error reading CSV file"
|
||||
});
|
||||
if (csvResults.length === 0) {
|
||||
deleteFileSecure(sanitizedPath);
|
||||
return sendUploadResponse(res, 400, {
|
||||
status: "failed",
|
||||
message: "CSV file is empty or invalid.",
|
||||
});
|
||||
}
|
||||
|
||||
const requiredCols = ["hs_code", "product_name", "unit"];
|
||||
const optionalCols = ["description", "weight_in_ib"];
|
||||
const allowedCols = requiredCols.concat(optionalCols);
|
||||
const headers = Object.keys(csvResults[0]);
|
||||
|
||||
const missingCols = requiredCols.filter((col) => !headers.includes(col));
|
||||
const extraCols = headers.filter((col) => !allowedCols.includes(col));
|
||||
|
||||
if (missingCols.length > 0 || extraCols.length > 0) {
|
||||
deleteFileSecure(sanitizedPath);
|
||||
return sendUploadResponse(res, 400, {
|
||||
status: "failed",
|
||||
message:
|
||||
(missingCols.length ? `Missing columns: ${missingCols.join(", ")}. ` : "") +
|
||||
(extraCols.length ? `Unexpected columns: ${extraCols.join(", ")}.` : ""),
|
||||
});
|
||||
}
|
||||
|
||||
const normalizedRows = [];
|
||||
const seenHsCodes = new Set();
|
||||
const fileDuplicates = new Set();
|
||||
const validationErrors = [];
|
||||
|
||||
for (let index = 0; index < csvResults.length; index++) {
|
||||
const row = csvResults[index];
|
||||
const csvLineNumber = index + 2;
|
||||
|
||||
if (isBlankCsvRow(row)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const hsCode = row.hs_code ? row.hs_code.replace(/[-\s/]/g, "").trim() : "";
|
||||
const productName = row.product_name
|
||||
? row.product_name.replace(/\s+/g, " ").trim()
|
||||
: "";
|
||||
const unit = row.unit ? row.unit.trim().toLowerCase() : "";
|
||||
const description = row.description ? row.description.trim() : "";
|
||||
|
||||
let weightInIb = null;
|
||||
if (
|
||||
row.weight_in_ib !== undefined &&
|
||||
row.weight_in_ib !== null &&
|
||||
String(row.weight_in_ib).trim() !== ""
|
||||
) {
|
||||
const wp = parseWeightInIb(row.weight_in_ib);
|
||||
if (!wp.ok) {
|
||||
validationErrors.push({ row: csvLineNumber, error: wp.message });
|
||||
continue;
|
||||
}
|
||||
weightInIb = wp.value;
|
||||
}
|
||||
|
||||
if (!hsCode || !productName || !unit) {
|
||||
const missing = [];
|
||||
if (!hsCode) missing.push("HS Code");
|
||||
if (!productName) missing.push("Product Name");
|
||||
if (!unit) missing.push("Unit");
|
||||
validationErrors.push({
|
||||
row: csvLineNumber,
|
||||
error: `Missing required field(s): ${missing.join(", ")}`,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!/^\d+$/.test(hsCode)) {
|
||||
validationErrors.push({ row: csvLineNumber, error: "HS Code must be numeric" });
|
||||
continue;
|
||||
}
|
||||
|
||||
if (hsCode.length > 10) {
|
||||
validationErrors.push({
|
||||
row: csvLineNumber,
|
||||
error: "HS Code must be max 10 digits",
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
if (hsCode === "0" || /^0+$/.test(hsCode)) {
|
||||
validationErrors.push({
|
||||
row: csvLineNumber,
|
||||
error: "HS Code cannot be 0",
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
if (productName.length > 1000) {
|
||||
validationErrors.push({
|
||||
row: csvLineNumber,
|
||||
error: "Product Name is too long. Maximum 1000 characters.",
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
if (description.length > 1000) {
|
||||
validationErrors.push({
|
||||
row: csvLineNumber,
|
||||
error: "HS Description is too long. Maximum 1000 characters.",
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
const normalizedHs = hsCode.replace(/^0+/, "") || hsCode;
|
||||
if (seenHsCodes.has(normalizedHs)) {
|
||||
fileDuplicates.add(hsCode);
|
||||
continue;
|
||||
}
|
||||
seenHsCodes.add(normalizedHs);
|
||||
|
||||
normalizedRows.push({
|
||||
hsCode,
|
||||
productName,
|
||||
unit,
|
||||
description,
|
||||
weightInIb,
|
||||
csvLineNumber,
|
||||
});
|
||||
}
|
||||
|
||||
if (normalizedRows.length === 0 && validationErrors.length === 0) {
|
||||
deleteFileSecure(sanitizedPath);
|
||||
return sendUploadResponse(res, 400, {
|
||||
status: "failed",
|
||||
message:
|
||||
"No product data rows found. Add data below the header row (remove blank rows from the template).",
|
||||
});
|
||||
}
|
||||
|
||||
if (fileDuplicates.size > 0) {
|
||||
deleteFileSecure(sanitizedPath);
|
||||
return sendUploadResponse(res, 400, {
|
||||
status: "failed",
|
||||
message:
|
||||
"Duplicate HS Codes found within file. Resolve and re-upload.",
|
||||
duplicate_hs_codes_in_file: Array.from(fileDuplicates),
|
||||
});
|
||||
}
|
||||
|
||||
const unitMasters = await UnitMaster.findAll({
|
||||
attributes: ["id", "uom", "uom_short_name"],
|
||||
where: { is_active: true },
|
||||
});
|
||||
const unitMap = buildUnitLookupMap(unitMasters);
|
||||
|
||||
const existingProducts = await Product.findAll({
|
||||
attributes: ["hs_code", "product_name"],
|
||||
where: { is_active: true },
|
||||
});
|
||||
|
||||
const existingHsCodeSet = new Set();
|
||||
const existingProductNameSet = new Set();
|
||||
for (const product of existingProducts) {
|
||||
if (product.hs_code != null && String(product.hs_code).trim() !== "") {
|
||||
existingHsCodeSet.add(String(product.hs_code).replace(/^0+/, "") || "0");
|
||||
}
|
||||
if (product.product_name) {
|
||||
existingProductNameSet.add(product.product_name.trim().toLowerCase());
|
||||
}
|
||||
}
|
||||
|
||||
const duplicateHsCodes = [];
|
||||
const duplicateProductNames = [];
|
||||
|
||||
for (let i = 0; i < normalizedRows.length; i++) {
|
||||
const row = normalizedRows[i];
|
||||
const normalizedHs = row.hsCode.replace(/^0+/, "") || row.hsCode;
|
||||
const productNameLower = row.productName.trim().toLowerCase();
|
||||
|
||||
if (existingHsCodeSet.has(normalizedHs)) {
|
||||
duplicateHsCodes.push({
|
||||
row: row.csvLineNumber,
|
||||
hs_code: row.hsCode,
|
||||
product_name: row.productName,
|
||||
});
|
||||
}
|
||||
|
||||
if (existingProductNameSet.has(productNameLower)) {
|
||||
duplicateProductNames.push({
|
||||
row: row.csvLineNumber,
|
||||
hs_code: row.hsCode,
|
||||
product_name: row.productName,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (duplicateHsCodes.length > 0 || duplicateProductNames.length > 0) {
|
||||
deleteFileSecure(sanitizedPath);
|
||||
const errorMessages = [];
|
||||
if (duplicateHsCodes.length > 0) {
|
||||
errorMessages.push(
|
||||
`${duplicateHsCodes.length} duplicate HS Code(s) found in database`
|
||||
);
|
||||
}
|
||||
if (duplicateProductNames.length > 0) {
|
||||
errorMessages.push(
|
||||
`${duplicateProductNames.length} duplicate Product Name(s) found in database`
|
||||
);
|
||||
}
|
||||
return sendUploadResponse(res, 400, {
|
||||
status: "failed",
|
||||
message:
|
||||
"Upload rejected: " +
|
||||
errorMessages.join(", ") +
|
||||
". Please remove duplicates and try again.",
|
||||
duplicate_hs_codes: duplicateHsCodes,
|
||||
duplicate_product_names: duplicateProductNames,
|
||||
total_duplicates:
|
||||
duplicateHsCodes.length + duplicateProductNames.length,
|
||||
});
|
||||
}
|
||||
|
||||
const toInsert = [];
|
||||
|
||||
for (let i = 0; i < normalizedRows.length; i++) {
|
||||
const row = normalizedRows[i];
|
||||
const unitId = unitMap.get(row.unit);
|
||||
|
||||
if (!unitId) {
|
||||
validationErrors.push({
|
||||
row: row.csvLineNumber,
|
||||
error: `Invalid unit "${row.unit}". Use unit short code (e.g. NO) or full name (e.g. Numbers) from Unit Master.`,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
toInsert.push({
|
||||
hs_code: parseInt(row.hsCode, 10),
|
||||
product_name: row.productName,
|
||||
unit_id: unitId,
|
||||
hs_description: row.description || null,
|
||||
weight_in_ib: row.weightInIb,
|
||||
created_by: userId,
|
||||
created_at: new Date(),
|
||||
is_active: true,
|
||||
});
|
||||
}
|
||||
|
||||
if (validationErrors.length > 0) {
|
||||
deleteFileSecure(sanitizedPath);
|
||||
return sendUploadResponse(res, 400, {
|
||||
status: "failed",
|
||||
message: `Upload rejected: ${validationErrors.length} validation error(s) found. Please fix and try again.`,
|
||||
errors: validationErrors,
|
||||
});
|
||||
}
|
||||
|
||||
if (toInsert.length === 0) {
|
||||
deleteFileSecure(sanitizedPath);
|
||||
return sendUploadResponse(res, 400, {
|
||||
status: "failed",
|
||||
message: "No valid product rows to import.",
|
||||
});
|
||||
}
|
||||
|
||||
let insertedRecords = [];
|
||||
try {
|
||||
insertedRecords = await Product.bulkCreate(toInsert, {
|
||||
validate: true,
|
||||
individualHooks: false,
|
||||
});
|
||||
} catch (bulkError) {
|
||||
deleteFileSecure(sanitizedPath);
|
||||
logger.error("Product bulkCreate error: " + bulkError.message);
|
||||
logger.error(`Stack trace: ${bulkError.stack}`);
|
||||
if (tryRespondProductPersistenceError(bulkError, res)) {
|
||||
return;
|
||||
}
|
||||
throw bulkError;
|
||||
}
|
||||
|
||||
deleteFileSecure(sanitizedPath);
|
||||
|
||||
return sendUploadResponse(res, 201, {
|
||||
status: "success",
|
||||
message: `${insertedRecords.length} product(s) inserted successfully.`,
|
||||
summary: {
|
||||
total_records: csvResults.length,
|
||||
imported: insertedRecords.length,
|
||||
skipped: csvResults.length - insertedRecords.length,
|
||||
errors: [],
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
deleteFileSecure(sanitizedPath);
|
||||
if (logger && logger.error) {
|
||||
logger.error("Upload error: " + error.message);
|
||||
logger.error(`Stack trace: ${error.stack}`);
|
||||
logger.error("Upload error: " + error.message);
|
||||
logger.error(`Stack trace: ${error.stack}`);
|
||||
if (tryRespondProductPersistenceError(error, res)) {
|
||||
return;
|
||||
}
|
||||
return res.status(500).send({
|
||||
status: "failed",
|
||||
message: "Internal server error"
|
||||
return sendUploadResponse(res, 500, {
|
||||
status: "failed",
|
||||
message: "Internal server error",
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
Loading…
Reference in New Issue
Block a user