hs code mapping done
This commit is contained in:
unknown
parent
aee1d48171
commit
f437d995c9
@ -9,7 +9,7 @@ const Emirate = db.Emirate;
|
|||||||
const user = db.user;
|
const user = db.user;
|
||||||
const Product = db.Product;
|
const Product = db.Product;
|
||||||
const ExcelJS = require("exceljs");
|
const ExcelJS = require("exceljs");
|
||||||
const { Op } = require("sequelize");
|
const { Op, Sequelize } = require("sequelize");
|
||||||
const { sendEmail } = require("../services/emailHelper");
|
const { sendEmail } = require("../services/emailHelper");
|
||||||
const { sendEmailService } = require("../services/email.service");
|
const { sendEmailService } = require("../services/email.service");
|
||||||
const logger = require("../services/logger");
|
const logger = require("../services/logger");
|
||||||
@ -819,128 +819,266 @@ exports.forgotPasswordVerifyOTP = async (req, res) => {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
exports.establishmentBulkUpload = async (req, res) => {
|
exports.establishmentBulkUpload = async (req, res) => {
|
||||||
|
const removeFile = (path) => fs.existsSync(path) && fs.unlinkSync(path);
|
||||||
|
const HS_CODE_LENGTH = 10;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// Check if file exists
|
if (!req.file) {
|
||||||
if (!req.file) {
|
return res.status(400).send({ status: "failed", message: "No file uploaded." });
|
||||||
return res.status(400).send({ status: "failed", message: "No file uploaded" });
|
}
|
||||||
}
|
|
||||||
|
|
||||||
const filePath = req.file.path;
|
|
||||||
|
|
||||||
// Validate file type
|
|
||||||
if (!req.file.originalname.endsWith(".csv")) {
|
|
||||||
fs.unlinkSync(filePath);
|
|
||||||
return res.status(400).send({
|
|
||||||
status: "failed",
|
|
||||||
message: "Invalid file type. Only CSV files are allowed.",
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
// Validate userId
|
|
||||||
if (!req.body.userId || isNaN(req.body.userId)) {
|
|
||||||
fs.unlinkSync(filePath);
|
|
||||||
return res.status(400).send({
|
|
||||||
status: "failed",
|
|
||||||
message: "Invalid or missing userId. Must be an integer.",
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
const results = [];
|
|
||||||
const userId = parseInt(req.body.userId);
|
|
||||||
|
|
||||||
// Read CSV and clean headers/values
|
const filePath = req.file.path;
|
||||||
fs.createReadStream(filePath)
|
if (!req.file.originalname.endsWith(".csv")) {
|
||||||
.pipe(csv())
|
removeFile(filePath);
|
||||||
.on("data", (row) => {
|
return res.status(400).send({
|
||||||
// Trim all keys and values to handle spaces in header names or values
|
status: "failed",
|
||||||
const cleanRow = {};
|
message: "Invalid file type. Only CSV allowed.",
|
||||||
for (const key in row) {
|
});
|
||||||
cleanRow[key.trim()] = row[key] ? row[key].trim() : null;
|
}
|
||||||
}
|
|
||||||
results.push(cleanRow);
|
|
||||||
})
|
|
||||||
.on("end", async () => {
|
|
||||||
try {
|
|
||||||
// Check for empty CSV
|
|
||||||
if (results.length === 0) {
|
|
||||||
fs.unlinkSync(filePath);
|
|
||||||
return res.status(400).send({
|
|
||||||
status: "failed",
|
|
||||||
message: "CSV file is empty or invalid.",
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
const estCode = results.map((r) => r.EstablishmentId);
|
|
||||||
const existingEstablishment = await Establishment.findAll({
|
|
||||||
where: { establishment_code: estCode },
|
|
||||||
});
|
|
||||||
|
|
||||||
const existingEstcode = existingEstablishment.map((p) => p.establishment_code);
|
|
||||||
const toInsert = [];
|
|
||||||
const duplicates = [];
|
|
||||||
|
|
||||||
// Process CSV rows
|
|
||||||
for (const row of results) {
|
|
||||||
const establishmentId = row.EstablishmentId?.trim();
|
|
||||||
const factoryName = row.FactoryName?.trim();
|
|
||||||
const email = row.Email?.trim();
|
|
||||||
const emirateName = row.Emirate?.trim();
|
|
||||||
const totalEmployment = row.TotalEmployment;
|
|
||||||
|
|
||||||
if (!establishmentId || !factoryName) continue; // skip invalid rows
|
|
||||||
|
|
||||||
const emirateRecord = await Emirate.findOne({
|
const mode = req.body.mode?.toLowerCase() || "add";
|
||||||
where: { name: emirateName },
|
if (mode !== "add") {
|
||||||
attributes: ['id']
|
removeFile(filePath);
|
||||||
});
|
return res.status(400).send({
|
||||||
const emirateId = emirateRecord ? emirateRecord.id : null;
|
status: "failed",
|
||||||
|
message: "Only 'Add Only' mode is supported currently.",
|
||||||
if (existingEstcode.includes(establishmentId)) {
|
});
|
||||||
duplicates.push(establishmentId);
|
}
|
||||||
} else {
|
|
||||||
toInsert.push({
|
const stats = fs.statSync(filePath);
|
||||||
establishment_code: establishmentId,
|
if (stats.size === 0) {
|
||||||
factory_name: factoryName,
|
removeFile(filePath);
|
||||||
establishment_contact_email: email,
|
return res.status(400).send({
|
||||||
establishment_emirate_id: emirateRecord ? emirateRecord.id : null,
|
status: "failed",
|
||||||
total_employees:totalEmployment,
|
message: "Uploaded file is empty.",
|
||||||
created_by: userId,
|
});
|
||||||
created_at: new Date(),
|
}
|
||||||
});
|
|
||||||
}
|
const results = [];
|
||||||
}
|
|
||||||
// Bulk insert new records
|
fs.createReadStream(filePath)
|
||||||
let inserted = [];
|
.pipe(csv())
|
||||||
if (toInsert.length > 0) {
|
.on("data", (row) => {
|
||||||
inserted = await Establishment.bulkCreate(toInsert, { validate: true });
|
const cleanRow = {};
|
||||||
}
|
for (const key in row) {
|
||||||
|
cleanRow[key.trim()] = row[key]?.trim() || null;
|
||||||
// Delete file after processing
|
}
|
||||||
fs.unlinkSync(filePath);
|
results.push(cleanRow);
|
||||||
|
})
|
||||||
// Send success response
|
.on("end", async () => {
|
||||||
return res.status(200).send({
|
try {
|
||||||
status: "success",
|
if (results.length === 0) {
|
||||||
message: '${inserted.length}Customer Profiles inserted.',
|
removeFile(filePath);
|
||||||
inserted_count: inserted.length,
|
return res.status(400).send({
|
||||||
duplicate_establishment: duplicates,
|
status: "failed",
|
||||||
});
|
message: "CSV file is empty or invalid.",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
const requiredCols = [
|
||||||
|
"Establishment Id",
|
||||||
|
"Factory Name",
|
||||||
|
"Email",
|
||||||
|
"Emirate",
|
||||||
|
"Total Employment",
|
||||||
|
];
|
||||||
|
|
||||||
|
const headers = Object.keys(results[0]);
|
||||||
|
const missingCols = requiredCols.filter((col) => !headers.includes(col));
|
||||||
|
const extraCols = headers.filter(
|
||||||
|
(col) => !requiredCols.includes(col) && !col.toLowerCase().startsWith("hs code")
|
||||||
|
);
|
||||||
|
|
||||||
|
if (missingCols.length > 0 || extraCols.length > 0) {
|
||||||
|
removeFile(filePath);
|
||||||
|
let message = "";
|
||||||
|
if (missingCols.length > 0)
|
||||||
|
message += `Missing required columns: ${missingCols.join(", ")}. `;
|
||||||
|
if (extraCols.length > 0)
|
||||||
|
message += `Unexpected columns found: ${extraCols.join(", ")}. Only 'Establishment Id', 'Factory Name', 'Email', 'Emirate', 'Total Employment', and HS Code columns are allowed.`;
|
||||||
|
return res.status(400).send({
|
||||||
|
status: "failed",
|
||||||
|
message: message.trim(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
const hsCodeCols = headers.filter((h) =>
|
||||||
|
h.toLowerCase().replace(/[\s_]+/g, "").startsWith("hscode")
|
||||||
|
);
|
||||||
|
|
||||||
|
const toInsert = [];
|
||||||
|
const duplicatesInFile = [];
|
||||||
|
const errors = [];
|
||||||
|
const seenKeys = new Set();
|
||||||
|
const seenEmails = new Set();
|
||||||
|
|
||||||
|
for (const [i, row] of results.entries()) {
|
||||||
|
const rowNum = i + 1;
|
||||||
|
const estId = row["Establishment Id"];
|
||||||
|
const factory = row["Factory Name"];
|
||||||
|
const email = row["Email"];
|
||||||
|
const emirate = row["Emirate"];
|
||||||
|
const emp = row["Total Employment"];
|
||||||
|
|
||||||
|
// Basic field check
|
||||||
|
if (!estId || !factory || !email || !emirate || !emp) {
|
||||||
|
errors.push({ row: rowNum, error: "Missing required fields." });
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Email format
|
||||||
|
const emailRegex = /^[^\s@]+@[^\s@]+\.[^\s@]+$/;
|
||||||
|
if (!emailRegex.test(email)) {
|
||||||
|
errors.push({ row: rowNum, error: `Invalid email: ${email}` });
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Total Employment numeric
|
||||||
|
if (isNaN(emp)) {
|
||||||
|
errors.push({ row: rowNum, error: "Total Employment must be a number." });
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
// File-level duplicates
|
||||||
|
const key = `${estId}|${factory}|${email}|${emirate}`.toLowerCase();
|
||||||
|
if (seenKeys.has(key)) {
|
||||||
|
duplicatesInFile.push({ row: rowNum, error: "Duplicate record in file." });
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
seenKeys.add(key);
|
||||||
|
|
||||||
|
if (seenEmails.has(email.toLowerCase())) {
|
||||||
|
duplicatesInFile.push({ row: rowNum, error: `Duplicate email in file: ${email}` });
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
seenEmails.add(email.toLowerCase());
|
||||||
|
|
||||||
|
// Emirate validation
|
||||||
|
const emirateRecord = await Emirate.findOne({
|
||||||
|
where: Sequelize.where(
|
||||||
|
Sequelize.fn("LOWER", Sequelize.col("name")),
|
||||||
|
emirate.toLowerCase()
|
||||||
|
),
|
||||||
|
attributes: ["id"],
|
||||||
|
});
|
||||||
|
if (!emirateRecord) {
|
||||||
|
errors.push({ row: rowNum, error: `Invalid Emirate: ${emirate}` });
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Extract and validate HS Codes
|
||||||
|
const hsCodes = hsCodeCols
|
||||||
|
.map((col) => row[col])
|
||||||
|
.filter(Boolean)
|
||||||
|
.flatMap((v) =>
|
||||||
|
v
|
||||||
|
.split(/[,\s]+/)
|
||||||
|
.map((x) => x.replace(/[-\/]/g, "").trim())
|
||||||
|
.filter((x) => /^\d+$/.test(x) && x.length === HS_CODE_LENGTH)
|
||||||
|
);
|
||||||
|
|
||||||
|
// DB duplicate check
|
||||||
|
const existingEst = await Establishment.findOne({
|
||||||
|
where: {
|
||||||
|
[Op.or]: [
|
||||||
|
{ establishment_code: estId },
|
||||||
|
{ factory_name: factory },
|
||||||
|
{ establishment_contact_email: email },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
attributes: ["establishment_code", "factory_name", "establishment_contact_email"],
|
||||||
|
});
|
||||||
|
|
||||||
|
if (existingEst) {
|
||||||
|
const details = [];
|
||||||
|
if (existingEst.establishment_code === estId)
|
||||||
|
details.push({
|
||||||
|
field: "establishment_code",
|
||||||
|
message: "Establishment Id already exists",
|
||||||
|
});
|
||||||
|
if (existingEst.factory_name === factory)
|
||||||
|
details.push({
|
||||||
|
field: "factory_name",
|
||||||
|
message: "Factory Name already exists",
|
||||||
|
});
|
||||||
|
if (existingEst.establishment_contact_email === email)
|
||||||
|
details.push({
|
||||||
|
field: "establishment_contact_email",
|
||||||
|
message: "Email already exists",
|
||||||
|
});
|
||||||
|
|
||||||
|
removeFile(filePath);
|
||||||
|
return res.status(400).send({
|
||||||
|
status: "failed",
|
||||||
|
message: "Validation error",
|
||||||
|
details,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Passed validation
|
||||||
|
toInsert.push({
|
||||||
|
establishment_code: estId,
|
||||||
|
factory_name: factory,
|
||||||
|
establishment_contact_email: email,
|
||||||
|
establishment_emirate_id: emirateRecord.id,
|
||||||
|
total_employees: parseInt(emp),
|
||||||
|
hs_codes: hsCodes,
|
||||||
|
created_by: req.user.id,
|
||||||
|
created_at: new Date(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stop if file duplicates found
|
||||||
|
if (duplicatesInFile.length > 0) {
|
||||||
|
removeFile(filePath);
|
||||||
|
return res.status(400).send({
|
||||||
|
status: "failed",
|
||||||
|
message: "Duplicate records found in file. Resolve and re-upload.",
|
||||||
|
duplicates: duplicatesInFile,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
const inserted = await Establishment.bulkCreate(toInsert, { validate: true });
|
||||||
|
|
||||||
|
removeFile(filePath);
|
||||||
|
|
||||||
|
let message = `${inserted.length} establishments inserted successfully.`;
|
||||||
|
let finalStatus = "success";
|
||||||
|
if (errors.length > 0) {
|
||||||
|
finalStatus = inserted.length > 0 ? "partial_success" : "failed";
|
||||||
|
message = `${inserted.length} inserted, ${errors.length} skipped due to errors.`;
|
||||||
|
}
|
||||||
|
|
||||||
|
return res.status(200).send({
|
||||||
|
status: finalStatus,
|
||||||
|
message,
|
||||||
|
summary: {
|
||||||
|
total_records: results.length,
|
||||||
|
imported: inserted.length,
|
||||||
|
errors: errors.length,
|
||||||
|
duplicates: duplicatesInFile.length,
|
||||||
|
},
|
||||||
|
errors,
|
||||||
|
});
|
||||||
|
} catch (err) {
|
||||||
|
removeFile(filePath);
|
||||||
|
console.error("Processing error:", err);
|
||||||
|
return res.status(500).send({
|
||||||
|
status: "failed",
|
||||||
|
message: err.message,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
console.error(error);
|
||||||
|
return res.status(500).send({
|
||||||
|
status: "failed",
|
||||||
|
message: error.message,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
} catch (err) {
|
|
||||||
if (fs.existsSync(filePath)) fs.unlinkSync(filePath);
|
|
||||||
return res.status(500).send({
|
|
||||||
status: "failed",
|
|
||||||
message: err.message,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
});
|
|
||||||
} catch (error) {
|
|
||||||
return res.status(500).send({ status: "failed", message: error.message });
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@ -668,7 +668,7 @@ router.delete("/establishments/:id",[verifySignature, verifyToken], establishmen
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* @swagger
|
* @swagger
|
||||||
* /api/establishments/uploadCSV:
|
* /api/company-profile/uploadCSV:
|
||||||
* post:
|
* post:
|
||||||
* summary: Upload Establishment data in bulk using CSV file
|
* summary: Upload Establishment data in bulk using CSV file
|
||||||
* description: This API accepts CSV file and inserts multiple Establishment in bulk. CSV header columns must match Establishment table columns.
|
* description: This API accepts CSV file and inserts multiple Establishment in bulk. CSV header columns must match Establishment table columns.
|
||||||
@ -695,7 +695,7 @@ router.delete("/establishments/:id",[verifySignature, verifyToken], establishmen
|
|||||||
* 500:
|
* 500:
|
||||||
* description: Server error
|
* description: Server error
|
||||||
*/
|
*/
|
||||||
router.post("/establishments/uploadCSV",[verifySignature, verifyToken, upload.single("file")], establishmentController.establishmentBulkUpload);
|
router.post("/company-profile/uploadCSV",[verifySignature, verifyToken, upload.single("file")], establishmentController.establishmentBulkUpload);
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user