Geetha : uploadProductsFromCSV
This commit is contained in:
parent
e36857b7bf
commit
80d12c068c
@ -101,122 +101,122 @@ exports.deleteProduct = async (req, res) => {
|
||||
}
|
||||
};
|
||||
|
||||
exports.uploadProductsFromCSV = async (req, res) => {
|
||||
try {
|
||||
// Check if file exists
|
||||
if (!req.file) {
|
||||
return res.status(400).send({ status: "failed", message: "No file uploaded" });
|
||||
}
|
||||
// exports.uploadProductsFromCSV = async (req, res) => {
|
||||
// try {
|
||||
// // Check if file exists
|
||||
// if (!req.file) {
|
||||
// return res.status(400).send({ status: "failed", message: "No file uploaded" });
|
||||
// }
|
||||
|
||||
const filePath = req.file.path;
|
||||
// const filePath = req.file.path;
|
||||
|
||||
// Validate file type
|
||||
if (!req.file.originalname.endsWith(".csv")) {
|
||||
fs.unlinkSync(filePath);
|
||||
return res.status(400).send({
|
||||
status: "failed",
|
||||
message: "Invalid file type. Only CSV files are allowed.",
|
||||
});
|
||||
}
|
||||
// // Validate file type
|
||||
// if (!req.file.originalname.endsWith(".csv")) {
|
||||
// fs.unlinkSync(filePath);
|
||||
// return res.status(400).send({
|
||||
// status: "failed",
|
||||
// message: "Invalid file type. Only CSV files are allowed.",
|
||||
// });
|
||||
// }
|
||||
|
||||
// Validate userId
|
||||
if (!req.body.userId || isNaN(req.body.userId)) {
|
||||
fs.unlinkSync(filePath);
|
||||
return res.status(400).send({
|
||||
status: "failed",
|
||||
message: "Invalid or missing userId. Must be an integer.",
|
||||
});
|
||||
}
|
||||
// // Validate userId
|
||||
// if (!req.body.userId || isNaN(req.body.userId)) {
|
||||
// fs.unlinkSync(filePath);
|
||||
// return res.status(400).send({
|
||||
// status: "failed",
|
||||
// message: "Invalid or missing userId. Must be an integer.",
|
||||
// });
|
||||
// }
|
||||
|
||||
const results = [];
|
||||
const userId = parseInt(req.body.userId);
|
||||
// const results = [];
|
||||
// const userId = parseInt(req.body.userId);
|
||||
|
||||
console.log(userId, "userId");
|
||||
// console.log(userId, "userId");
|
||||
|
||||
// Read CSV and clean headers/values
|
||||
fs.createReadStream(filePath)
|
||||
.pipe(csv())
|
||||
.on("data", (row) => {
|
||||
// Trim all keys and values to handle spaces in header names or values
|
||||
const cleanRow = {};
|
||||
for (const key in row) {
|
||||
cleanRow[key.trim()] = row[key] ? row[key].trim() : null;
|
||||
}
|
||||
results.push(cleanRow);
|
||||
})
|
||||
.on("end", async () => {
|
||||
try {
|
||||
// Check for empty CSV
|
||||
if (results.length === 0) {
|
||||
fs.unlinkSync(filePath);
|
||||
return res.status(400).send({
|
||||
status: "failed",
|
||||
message: "CSV file is empty or invalid.",
|
||||
});
|
||||
}
|
||||
// // Read CSV and clean headers/values
|
||||
// fs.createReadStream(filePath)
|
||||
// .pipe(csv())
|
||||
// .on("data", (row) => {
|
||||
// // Trim all keys and values to handle spaces in header names or values
|
||||
// const cleanRow = {};
|
||||
// for (const key in row) {
|
||||
// cleanRow[key.trim()] = row[key] ? row[key].trim() : null;
|
||||
// }
|
||||
// results.push(cleanRow);
|
||||
// })
|
||||
// .on("end", async () => {
|
||||
// try {
|
||||
// // Check for empty CSV
|
||||
// if (results.length === 0) {
|
||||
// fs.unlinkSync(filePath);
|
||||
// return res.status(400).send({
|
||||
// status: "failed",
|
||||
// message: "CSV file is empty or invalid.",
|
||||
// });
|
||||
// }
|
||||
|
||||
console.log(results, "results");
|
||||
// console.log(results, "results");
|
||||
|
||||
// Fetch existing products by hs_code
|
||||
const hsCodes = results.map((r) => r.hs_code);
|
||||
const existingProducts = await Product.findAll({
|
||||
where: { hs_code: hsCodes },
|
||||
});
|
||||
// // Fetch existing products by hs_code
|
||||
// const hsCodes = results.map((r) => r.hs_code);
|
||||
// const existingProducts = await Product.findAll({
|
||||
// where: { hs_code: hsCodes },
|
||||
// });
|
||||
|
||||
const existingHsCodes = existingProducts.map((p) => p.hs_code);
|
||||
// const existingHsCodes = existingProducts.map((p) => p.hs_code);
|
||||
|
||||
const toInsert = [];
|
||||
const duplicates = [];
|
||||
// const toInsert = [];
|
||||
// const duplicates = [];
|
||||
|
||||
// Process CSV rows
|
||||
for (const row of results) {
|
||||
const hsCode = row.hs_code?.trim();
|
||||
const productName = row.product_name?.trim();
|
||||
const description = row.hs_description?.trim();
|
||||
// // Process CSV rows
|
||||
// for (const row of results) {
|
||||
// const hsCode = row.hs_code?.trim();
|
||||
// const productName = row.product_name?.trim();
|
||||
// const description = row.hs_description?.trim();
|
||||
|
||||
if (!hsCode || !productName) continue; // skip invalid rows
|
||||
// if (!hsCode || !productName) continue; // skip invalid rows
|
||||
|
||||
if (existingHsCodes.includes(hsCode)) {
|
||||
duplicates.push(hsCode);
|
||||
} else {
|
||||
toInsert.push({
|
||||
hs_code: hsCode,
|
||||
product_name: productName,
|
||||
hs_description: description,
|
||||
created_by: userId,
|
||||
created_at: new Date(),
|
||||
});
|
||||
}
|
||||
}
|
||||
// if (existingHsCodes.includes(hsCode)) {
|
||||
// duplicates.push(hsCode);
|
||||
// } else {
|
||||
// toInsert.push({
|
||||
// hs_code: hsCode,
|
||||
// product_name: productName,
|
||||
// hs_description: description,
|
||||
// created_by: userId,
|
||||
// created_at: new Date(),
|
||||
// });
|
||||
// }
|
||||
// }
|
||||
|
||||
// Bulk insert new records
|
||||
let inserted = [];
|
||||
if (toInsert.length > 0) {
|
||||
inserted = await Product.bulkCreate(toInsert, { validate: true });
|
||||
}
|
||||
// // Bulk insert new records
|
||||
// let inserted = [];
|
||||
// if (toInsert.length > 0) {
|
||||
// inserted = await Product.bulkCreate(toInsert, { validate: true });
|
||||
// }
|
||||
|
||||
// Delete file after processing
|
||||
fs.unlinkSync(filePath);
|
||||
// // Delete file after processing
|
||||
// fs.unlinkSync(filePath);
|
||||
|
||||
// Send success response
|
||||
return res.status(200).send({
|
||||
status: "success",
|
||||
message: `${inserted.length} products inserted successfully.`,
|
||||
inserted_count: inserted.length,
|
||||
duplicate_hs_codes: duplicates,
|
||||
});
|
||||
} catch (err) {
|
||||
if (fs.existsSync(filePath)) fs.unlinkSync(filePath);
|
||||
return res.status(500).send({
|
||||
status: "failed",
|
||||
message: err.message,
|
||||
});
|
||||
}
|
||||
});
|
||||
} catch (error) {
|
||||
return res.status(500).send({ status: "failed", message: error.message });
|
||||
}
|
||||
};
|
||||
// // Send success response
|
||||
// return res.status(200).send({
|
||||
// status: "success",
|
||||
// message: `${inserted.length} products inserted successfully.`,
|
||||
// inserted_count: inserted.length,
|
||||
// duplicate_hs_codes: duplicates,
|
||||
// });
|
||||
// } catch (err) {
|
||||
// if (fs.existsSync(filePath)) fs.unlinkSync(filePath);
|
||||
// return res.status(500).send({
|
||||
// status: "failed",
|
||||
// message: err.message,
|
||||
// });
|
||||
// }
|
||||
// });
|
||||
// } catch (error) {
|
||||
// return res.status(500).send({ status: "failed", message: error.message });
|
||||
// }
|
||||
// };
|
||||
|
||||
exports.downloadProductSample = async (req, res) => {
|
||||
try {
|
||||
@ -226,3 +226,201 @@ exports.downloadProductSample = async (req, res) => {
|
||||
return res.status(500).send({ status: "failed", message: error.message });
|
||||
}
|
||||
};
|
||||
|
||||
exports.uploadProductsFromCSV = async (req, res) => {
|
||||
try {
|
||||
if (!req.file) {
|
||||
return res.status(400).send({ status: "failed", message: "No file uploaded" });
|
||||
}
|
||||
|
||||
const filePath = req.file.path;
|
||||
|
||||
if (!req.file.originalname.endsWith(".csv")) {
|
||||
fs.unlinkSync(filePath);
|
||||
return res.status(400).send({ status: "failed", message: "Invalid file type. Only CSV allowed." });
|
||||
}
|
||||
|
||||
if (!req.body.userId || isNaN(req.body.userId)) {
|
||||
fs.unlinkSync(filePath);
|
||||
return res.status(400).send({ status: "failed", message: "Invalid or missing userId." });
|
||||
}
|
||||
|
||||
const mode = req.body.mode?.toLowerCase() || "add";
|
||||
if (mode !== "add") {
|
||||
fs.unlinkSync(filePath);
|
||||
return res.status(400).send({ status: "failed", message: "Only 'Add Only' mode is supported currently." });
|
||||
}
|
||||
|
||||
const results = [];
|
||||
const userId = parseInt(req.body.userId);
|
||||
|
||||
const stats = fs.statSync(filePath);
|
||||
if (stats.size === 0) {
|
||||
fs.unlinkSync(filePath);
|
||||
return res.status(400).send({
|
||||
status: "failed",
|
||||
message: "Uploaded file is empty.",
|
||||
});
|
||||
}
|
||||
|
||||
fs.createReadStream(filePath)
|
||||
.pipe(csv())
|
||||
.on("data", (row) => {
|
||||
const cleanRow = {};
|
||||
|
||||
// Normalize headers and smart-map them
|
||||
for (const key in row) {
|
||||
const normalizedKey = key.replace(/[\s\W]+/g, "_").trim().toLowerCase();
|
||||
|
||||
let mappedKey = normalizedKey;
|
||||
if (/^hs.*code$/.test(normalizedKey) || normalizedKey === "hscode") {
|
||||
mappedKey = "hs_code";
|
||||
} else if (/^product.*name$/.test(normalizedKey) || /^produc.*tname$/.test(normalizedKey)) {
|
||||
mappedKey = "product_name";
|
||||
} else if (/^unit[s]?$/.test(normalizedKey) || /measurement|uom|measure/i.test(normalizedKey)) {
|
||||
mappedKey = "unit";
|
||||
}
|
||||
|
||||
cleanRow[mappedKey] = row[key]?.trim() || null;
|
||||
}
|
||||
|
||||
results.push(cleanRow);
|
||||
})
|
||||
.on("end", async () => {
|
||||
try {
|
||||
if (results.length === 0) {
|
||||
fs.unlinkSync(filePath);
|
||||
return res.status(400).send({ status: "failed", message: "CSV file is empty or invalid." });
|
||||
}
|
||||
|
||||
const requiredCols = ["hs_code", "product_name", "unit"];
|
||||
const headers = Object.keys(results[0]);
|
||||
|
||||
const missingCols = requiredCols.filter((col) => !headers.includes(col));
|
||||
const extraCols = headers.filter((col) => !requiredCols.includes(col));
|
||||
|
||||
if (missingCols.length > 0 || extraCols.length > 0) {
|
||||
fs.unlinkSync(filePath);
|
||||
let message = "";
|
||||
if (missingCols.length > 0) message += `Missing required columns: ${missingCols.join(", ")}. `;
|
||||
if (extraCols.length > 0) message += `Unexpected columns found: ${extraCols.join(", ")}. Only 'HS Code', 'Product Name', and 'Unit' are allowed.`;
|
||||
return res.status(400).send({ status: "failed", message: message.trim() });
|
||||
}
|
||||
|
||||
// Validate and normalize data
|
||||
const normalizedRows = [];
|
||||
const fileDuplicates = new Set();
|
||||
const seenHsCodes = new Set();
|
||||
const errors = [];
|
||||
|
||||
for (let [index, row] of results.entries()) {
|
||||
let hsCode = row.hs_code?.replace(/[-\s/]/g, "").trim();
|
||||
const productName = row.product_name?.replace(/\s+/g, " ").trim();
|
||||
const unit = row.unit?.trim().toLowerCase();
|
||||
|
||||
if (!hsCode || !productName) {
|
||||
errors.push({ row: index + 1, error: "Missing required HS Code or Product Name" });
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!/^\d+$/.test(hsCode)) {
|
||||
errors.push({ row: index + 1, error: "HS Code must be numeric" });
|
||||
continue;
|
||||
}
|
||||
|
||||
if (hsCode.length !== 10) {
|
||||
errors.push({ row: index + 1, error: "HS Code must be 10 digits" });
|
||||
continue;
|
||||
}
|
||||
|
||||
const normalizedHs = hsCode.replace(/^0+/, "");
|
||||
if (seenHsCodes.has(normalizedHs)) {
|
||||
fileDuplicates.add(hsCode);
|
||||
continue;
|
||||
}
|
||||
seenHsCodes.add(normalizedHs);
|
||||
|
||||
normalizedRows.push({ hsCode, productName, unit });
|
||||
}
|
||||
|
||||
if (fileDuplicates.size > 0) {
|
||||
fs.unlinkSync(filePath);
|
||||
return res.status(400).send({
|
||||
status: "failed",
|
||||
message: "Duplicate HS Codes found within file. Resolve and re-upload.",
|
||||
duplicate_hs_codes_in_file: [...fileDuplicates],
|
||||
});
|
||||
}
|
||||
|
||||
const unitMasters = await UnitMaster.findAll({ attributes: ["id", "uom"] });
|
||||
const unitMap = {};
|
||||
unitMasters.forEach((u) => (unitMap[u.uom.trim().toLowerCase()] = u.id));
|
||||
|
||||
const existingProducts = await Product.findAll({ attributes: ["hs_code"] });
|
||||
const existingSet = new Set(existingProducts.map((p) => p.hs_code.replace(/^0+/, "")));
|
||||
|
||||
const toInsert = [];
|
||||
const duplicates = [];
|
||||
|
||||
for (const row of normalizedRows) {
|
||||
const normalizedHs = row.hsCode.replace(/^0+/, "");
|
||||
if (existingSet.has(normalizedHs)) {
|
||||
duplicates.push(row.hsCode);
|
||||
continue;
|
||||
}
|
||||
|
||||
const unitId = unitMap[row.unit] || null;
|
||||
if (!unitId) {
|
||||
errors.push({ hs_code: row.hsCode, error: "Invalid unit" });
|
||||
continue;
|
||||
}
|
||||
|
||||
toInsert.push({
|
||||
hs_code: row.hsCode,
|
||||
product_name: row.productName,
|
||||
unit_id: unitId,
|
||||
created_by: userId,
|
||||
created_at: new Date(),
|
||||
});
|
||||
}
|
||||
|
||||
let inserted = [];
|
||||
if (toInsert.length > 0) {
|
||||
inserted = await Product.bulkCreate(toInsert, { validate: true });
|
||||
}
|
||||
|
||||
fs.unlinkSync(filePath);
|
||||
|
||||
let finalStatus = "success";
|
||||
let message = `${inserted.length} products inserted successfully.`;
|
||||
|
||||
if (errors.length > 0 || duplicates.length > 0) {
|
||||
finalStatus = inserted.length > 0 ? "partial_success" : "failed";
|
||||
if (finalStatus === "partial_success") {
|
||||
message = `${inserted.length} products inserted, ${duplicates.length} duplicates skipped, ${errors.length} validation errors found.`;
|
||||
} else {
|
||||
message = `No products imported. ${duplicates.length} duplicates and ${errors.length} validation errors found.`;
|
||||
}
|
||||
}
|
||||
|
||||
return res.status(200).send({
|
||||
status: finalStatus,
|
||||
message,
|
||||
summary: {
|
||||
total_records: results.length,
|
||||
imported: inserted.length,
|
||||
skipped: duplicates.length,
|
||||
errors: errors,
|
||||
},
|
||||
duplicate_hs_codes_in_system: duplicates,
|
||||
});
|
||||
} catch (err) {
|
||||
if (fs.existsSync(filePath)) fs.unlinkSync(filePath);
|
||||
return res.status(500).send({ status: "failed", message: err.message });
|
||||
}
|
||||
});
|
||||
} catch (error) {
|
||||
return res.status(500).send({ status: "failed", message: error.message });
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user