hs code mapping done
This commit is contained in:
unknown
parent
aee1d48171
commit
f437d995c9
@ -9,7 +9,7 @@ const Emirate = db.Emirate;
|
||||
const user = db.user;
|
||||
const Product = db.Product;
|
||||
const ExcelJS = require("exceljs");
|
||||
const { Op } = require("sequelize");
|
||||
const { Op, Sequelize } = require("sequelize");
|
||||
const { sendEmail } = require("../services/emailHelper");
|
||||
const { sendEmailService } = require("../services/email.service");
|
||||
const logger = require("../services/logger");
|
||||
@ -819,128 +819,266 @@ exports.forgotPasswordVerifyOTP = async (req, res) => {
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
exports.establishmentBulkUpload = async (req, res) => {
|
||||
const removeFile = (path) => fs.existsSync(path) && fs.unlinkSync(path);
|
||||
const HS_CODE_LENGTH = 10;
|
||||
|
||||
try {
|
||||
// Check if file exists
|
||||
if (!req.file) {
|
||||
return res.status(400).send({ status: "failed", message: "No file uploaded" });
|
||||
}
|
||||
|
||||
const filePath = req.file.path;
|
||||
|
||||
// Validate file type
|
||||
if (!req.file.originalname.endsWith(".csv")) {
|
||||
fs.unlinkSync(filePath);
|
||||
return res.status(400).send({
|
||||
status: "failed",
|
||||
message: "Invalid file type. Only CSV files are allowed.",
|
||||
});
|
||||
}
|
||||
|
||||
// Validate userId
|
||||
if (!req.body.userId || isNaN(req.body.userId)) {
|
||||
fs.unlinkSync(filePath);
|
||||
return res.status(400).send({
|
||||
status: "failed",
|
||||
message: "Invalid or missing userId. Must be an integer.",
|
||||
});
|
||||
}
|
||||
|
||||
const results = [];
|
||||
const userId = parseInt(req.body.userId);
|
||||
if (!req.file) {
|
||||
return res.status(400).send({ status: "failed", message: "No file uploaded." });
|
||||
}
|
||||
|
||||
// Read CSV and clean headers/values
|
||||
fs.createReadStream(filePath)
|
||||
.pipe(csv())
|
||||
.on("data", (row) => {
|
||||
// Trim all keys and values to handle spaces in header names or values
|
||||
const cleanRow = {};
|
||||
for (const key in row) {
|
||||
cleanRow[key.trim()] = row[key] ? row[key].trim() : null;
|
||||
}
|
||||
results.push(cleanRow);
|
||||
})
|
||||
.on("end", async () => {
|
||||
try {
|
||||
// Check for empty CSV
|
||||
if (results.length === 0) {
|
||||
fs.unlinkSync(filePath);
|
||||
return res.status(400).send({
|
||||
status: "failed",
|
||||
message: "CSV file is empty or invalid.",
|
||||
});
|
||||
}
|
||||
|
||||
const estCode = results.map((r) => r.EstablishmentId);
|
||||
const existingEstablishment = await Establishment.findAll({
|
||||
where: { establishment_code: estCode },
|
||||
});
|
||||
|
||||
const existingEstcode = existingEstablishment.map((p) => p.establishment_code);
|
||||
const toInsert = [];
|
||||
const duplicates = [];
|
||||
|
||||
// Process CSV rows
|
||||
for (const row of results) {
|
||||
const establishmentId = row.EstablishmentId?.trim();
|
||||
const factoryName = row.FactoryName?.trim();
|
||||
const email = row.Email?.trim();
|
||||
const emirateName = row.Emirate?.trim();
|
||||
const totalEmployment = row.TotalEmployment;
|
||||
|
||||
if (!establishmentId || !factoryName) continue; // skip invalid rows
|
||||
const filePath = req.file.path;
|
||||
if (!req.file.originalname.endsWith(".csv")) {
|
||||
removeFile(filePath);
|
||||
return res.status(400).send({
|
||||
status: "failed",
|
||||
message: "Invalid file type. Only CSV allowed.",
|
||||
});
|
||||
}
|
||||
|
||||
const emirateRecord = await Emirate.findOne({
|
||||
where: { name: emirateName },
|
||||
attributes: ['id']
|
||||
});
|
||||
const emirateId = emirateRecord ? emirateRecord.id : null;
|
||||
|
||||
if (existingEstcode.includes(establishmentId)) {
|
||||
duplicates.push(establishmentId);
|
||||
} else {
|
||||
toInsert.push({
|
||||
establishment_code: establishmentId,
|
||||
factory_name: factoryName,
|
||||
establishment_contact_email: email,
|
||||
establishment_emirate_id: emirateRecord ? emirateRecord.id : null,
|
||||
total_employees:totalEmployment,
|
||||
created_by: userId,
|
||||
created_at: new Date(),
|
||||
});
|
||||
}
|
||||
}
|
||||
// Bulk insert new records
|
||||
let inserted = [];
|
||||
if (toInsert.length > 0) {
|
||||
inserted = await Establishment.bulkCreate(toInsert, { validate: true });
|
||||
}
|
||||
|
||||
// Delete file after processing
|
||||
fs.unlinkSync(filePath);
|
||||
|
||||
// Send success response
|
||||
return res.status(200).send({
|
||||
status: "success",
|
||||
message: '${inserted.length}Customer Profiles inserted.',
|
||||
inserted_count: inserted.length,
|
||||
duplicate_establishment: duplicates,
|
||||
});
|
||||
const mode = req.body.mode?.toLowerCase() || "add";
|
||||
if (mode !== "add") {
|
||||
removeFile(filePath);
|
||||
return res.status(400).send({
|
||||
status: "failed",
|
||||
message: "Only 'Add Only' mode is supported currently.",
|
||||
});
|
||||
}
|
||||
|
||||
const stats = fs.statSync(filePath);
|
||||
if (stats.size === 0) {
|
||||
removeFile(filePath);
|
||||
return res.status(400).send({
|
||||
status: "failed",
|
||||
message: "Uploaded file is empty.",
|
||||
});
|
||||
}
|
||||
|
||||
const results = [];
|
||||
|
||||
fs.createReadStream(filePath)
|
||||
.pipe(csv())
|
||||
.on("data", (row) => {
|
||||
const cleanRow = {};
|
||||
for (const key in row) {
|
||||
cleanRow[key.trim()] = row[key]?.trim() || null;
|
||||
}
|
||||
results.push(cleanRow);
|
||||
})
|
||||
.on("end", async () => {
|
||||
try {
|
||||
if (results.length === 0) {
|
||||
removeFile(filePath);
|
||||
return res.status(400).send({
|
||||
status: "failed",
|
||||
message: "CSV file is empty or invalid.",
|
||||
});
|
||||
}
|
||||
|
||||
const requiredCols = [
|
||||
"Establishment Id",
|
||||
"Factory Name",
|
||||
"Email",
|
||||
"Emirate",
|
||||
"Total Employment",
|
||||
];
|
||||
|
||||
const headers = Object.keys(results[0]);
|
||||
const missingCols = requiredCols.filter((col) => !headers.includes(col));
|
||||
const extraCols = headers.filter(
|
||||
(col) => !requiredCols.includes(col) && !col.toLowerCase().startsWith("hs code")
|
||||
);
|
||||
|
||||
if (missingCols.length > 0 || extraCols.length > 0) {
|
||||
removeFile(filePath);
|
||||
let message = "";
|
||||
if (missingCols.length > 0)
|
||||
message += `Missing required columns: ${missingCols.join(", ")}. `;
|
||||
if (extraCols.length > 0)
|
||||
message += `Unexpected columns found: ${extraCols.join(", ")}. Only 'Establishment Id', 'Factory Name', 'Email', 'Emirate', 'Total Employment', and HS Code columns are allowed.`;
|
||||
return res.status(400).send({
|
||||
status: "failed",
|
||||
message: message.trim(),
|
||||
});
|
||||
}
|
||||
|
||||
const hsCodeCols = headers.filter((h) =>
|
||||
h.toLowerCase().replace(/[\s_]+/g, "").startsWith("hscode")
|
||||
);
|
||||
|
||||
const toInsert = [];
|
||||
const duplicatesInFile = [];
|
||||
const errors = [];
|
||||
const seenKeys = new Set();
|
||||
const seenEmails = new Set();
|
||||
|
||||
for (const [i, row] of results.entries()) {
|
||||
const rowNum = i + 1;
|
||||
const estId = row["Establishment Id"];
|
||||
const factory = row["Factory Name"];
|
||||
const email = row["Email"];
|
||||
const emirate = row["Emirate"];
|
||||
const emp = row["Total Employment"];
|
||||
|
||||
// Basic field check
|
||||
if (!estId || !factory || !email || !emirate || !emp) {
|
||||
errors.push({ row: rowNum, error: "Missing required fields." });
|
||||
continue;
|
||||
}
|
||||
|
||||
// Email format
|
||||
const emailRegex = /^[^\s@]+@[^\s@]+\.[^\s@]+$/;
|
||||
if (!emailRegex.test(email)) {
|
||||
errors.push({ row: rowNum, error: `Invalid email: ${email}` });
|
||||
continue;
|
||||
}
|
||||
|
||||
// Total Employment numeric
|
||||
if (isNaN(emp)) {
|
||||
errors.push({ row: rowNum, error: "Total Employment must be a number." });
|
||||
continue;
|
||||
}
|
||||
|
||||
// File-level duplicates
|
||||
const key = `${estId}|${factory}|${email}|${emirate}`.toLowerCase();
|
||||
if (seenKeys.has(key)) {
|
||||
duplicatesInFile.push({ row: rowNum, error: "Duplicate record in file." });
|
||||
continue;
|
||||
}
|
||||
seenKeys.add(key);
|
||||
|
||||
if (seenEmails.has(email.toLowerCase())) {
|
||||
duplicatesInFile.push({ row: rowNum, error: `Duplicate email in file: ${email}` });
|
||||
continue;
|
||||
}
|
||||
seenEmails.add(email.toLowerCase());
|
||||
|
||||
// Emirate validation
|
||||
const emirateRecord = await Emirate.findOne({
|
||||
where: Sequelize.where(
|
||||
Sequelize.fn("LOWER", Sequelize.col("name")),
|
||||
emirate.toLowerCase()
|
||||
),
|
||||
attributes: ["id"],
|
||||
});
|
||||
if (!emirateRecord) {
|
||||
errors.push({ row: rowNum, error: `Invalid Emirate: ${emirate}` });
|
||||
continue;
|
||||
}
|
||||
|
||||
// Extract and validate HS Codes
|
||||
const hsCodes = hsCodeCols
|
||||
.map((col) => row[col])
|
||||
.filter(Boolean)
|
||||
.flatMap((v) =>
|
||||
v
|
||||
.split(/[,\s]+/)
|
||||
.map((x) => x.replace(/[-\/]/g, "").trim())
|
||||
.filter((x) => /^\d+$/.test(x) && x.length === HS_CODE_LENGTH)
|
||||
);
|
||||
|
||||
// DB duplicate check
|
||||
const existingEst = await Establishment.findOne({
|
||||
where: {
|
||||
[Op.or]: [
|
||||
{ establishment_code: estId },
|
||||
{ factory_name: factory },
|
||||
{ establishment_contact_email: email },
|
||||
],
|
||||
},
|
||||
attributes: ["establishment_code", "factory_name", "establishment_contact_email"],
|
||||
});
|
||||
|
||||
if (existingEst) {
|
||||
const details = [];
|
||||
if (existingEst.establishment_code === estId)
|
||||
details.push({
|
||||
field: "establishment_code",
|
||||
message: "Establishment Id already exists",
|
||||
});
|
||||
if (existingEst.factory_name === factory)
|
||||
details.push({
|
||||
field: "factory_name",
|
||||
message: "Factory Name already exists",
|
||||
});
|
||||
if (existingEst.establishment_contact_email === email)
|
||||
details.push({
|
||||
field: "establishment_contact_email",
|
||||
message: "Email already exists",
|
||||
});
|
||||
|
||||
removeFile(filePath);
|
||||
return res.status(400).send({
|
||||
status: "failed",
|
||||
message: "Validation error",
|
||||
details,
|
||||
});
|
||||
}
|
||||
|
||||
// Passed validation
|
||||
toInsert.push({
|
||||
establishment_code: estId,
|
||||
factory_name: factory,
|
||||
establishment_contact_email: email,
|
||||
establishment_emirate_id: emirateRecord.id,
|
||||
total_employees: parseInt(emp),
|
||||
hs_codes: hsCodes,
|
||||
created_by: req.user.id,
|
||||
created_at: new Date(),
|
||||
});
|
||||
}
|
||||
|
||||
// Stop if file duplicates found
|
||||
if (duplicatesInFile.length > 0) {
|
||||
removeFile(filePath);
|
||||
return res.status(400).send({
|
||||
status: "failed",
|
||||
message: "Duplicate records found in file. Resolve and re-upload.",
|
||||
duplicates: duplicatesInFile,
|
||||
});
|
||||
}
|
||||
|
||||
const inserted = await Establishment.bulkCreate(toInsert, { validate: true });
|
||||
|
||||
removeFile(filePath);
|
||||
|
||||
let message = `${inserted.length} establishments inserted successfully.`;
|
||||
let finalStatus = "success";
|
||||
if (errors.length > 0) {
|
||||
finalStatus = inserted.length > 0 ? "partial_success" : "failed";
|
||||
message = `${inserted.length} inserted, ${errors.length} skipped due to errors.`;
|
||||
}
|
||||
|
||||
return res.status(200).send({
|
||||
status: finalStatus,
|
||||
message,
|
||||
summary: {
|
||||
total_records: results.length,
|
||||
imported: inserted.length,
|
||||
errors: errors.length,
|
||||
duplicates: duplicatesInFile.length,
|
||||
},
|
||||
errors,
|
||||
});
|
||||
} catch (err) {
|
||||
removeFile(filePath);
|
||||
console.error("Processing error:", err);
|
||||
return res.status(500).send({
|
||||
status: "failed",
|
||||
message: err.message,
|
||||
});
|
||||
}
|
||||
});
|
||||
} catch (error) {
|
||||
console.error(error);
|
||||
return res.status(500).send({
|
||||
status: "failed",
|
||||
message: error.message,
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
} catch (err) {
|
||||
if (fs.existsSync(filePath)) fs.unlinkSync(filePath);
|
||||
return res.status(500).send({
|
||||
status: "failed",
|
||||
message: err.message,
|
||||
});
|
||||
}
|
||||
});
|
||||
} catch (error) {
|
||||
return res.status(500).send({ status: "failed", message: error.message });
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
|
||||
|
||||
@ -668,7 +668,7 @@ router.delete("/establishments/:id",[verifySignature, verifyToken], establishmen
|
||||
|
||||
/**
|
||||
* @swagger
|
||||
* /api/establishments/uploadCSV:
|
||||
* /api/company-profile/uploadCSV:
|
||||
* post:
|
||||
* summary: Upload Establishment data in bulk using CSV file
|
||||
* description: This API accepts CSV file and inserts multiple Establishment in bulk. CSV header columns must match Establishment table columns.
|
||||
@ -695,7 +695,7 @@ router.delete("/establishments/:id",[verifySignature, verifyToken], establishmen
|
||||
* 500:
|
||||
* description: Server error
|
||||
*/
|
||||
router.post("/establishments/uploadCSV",[verifySignature, verifyToken, upload.single("file")], establishmentController.establishmentBulkUpload);
|
||||
router.post("/company-profile/uploadCSV",[verifySignature, verifyToken, upload.single("file")], establishmentController.establishmentBulkUpload);
|
||||
|
||||
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user