Column validation added

This commit is contained in:
unknown 2025-11-14 18:41:16 +05:30
parent 8ca289ad3f
commit a86286b54e
2 changed files with 320 additions and 195 deletions

View File

@ -559,8 +559,6 @@ exports.updateEstablishment = async (req, res) => {
};
// exports.updateEstablishment = async (req, res) => {
// try {
@ -820,265 +818,392 @@ exports.forgotPasswordVerifyOTP = async (req, res) => {
};
exports.establishmentBulkUpload = async (req, res) => {
const removeFile = (path) => fs.existsSync(path) && fs.unlinkSync(path);
const HS_CODE_LENGTH = 10;
try {
// Basic file & user validations
if (!req.file) {
return res.status(400).send({ status: "failed", message: "No file uploaded." });
return res.status(400).send({ status: "failed", message: "No file uploaded" });
}
const filePath = req.file.path;
if (!req.file.originalname.endsWith(".csv")) {
removeFile(filePath);
return res.status(400).send({
status: "failed",
message: "Invalid file type. Only CSV allowed.",
});
if (!req.file.originalname.toLowerCase().endsWith(".csv")) {
if (fs.existsSync(filePath)) fs.unlinkSync(filePath);
return res.status(400).send({ status: "failed", message: "Invalid file type. Only CSV allowed." });
}
const mode = req.body.mode?.toLowerCase() || "add";
if (!req.user?.id || isNaN(req.user.id)) {
if (fs.existsSync(filePath)) fs.unlinkSync(filePath);
return res.status(400).send({ status: "failed", message: "Invalid or missing User Id." });
}
const mode = (req.body.mode || "add").toLowerCase();
if (mode !== "add") {
removeFile(filePath);
return res.status(400).send({
status: "failed",
message: "Only 'Add Only' mode is supported currently.",
});
if (fs.existsSync(filePath)) fs.unlinkSync(filePath);
return res.status(400).send({ status: "failed", message: "Only 'Add Only' mode is supported currently." });
}
const stats = fs.statSync(filePath);
if (stats.size === 0) {
removeFile(filePath);
return res.status(400).send({
status: "failed",
message: "Uploaded file is empty.",
});
if (fs.existsSync(filePath)) fs.unlinkSync(filePath);
return res.status(400).send({ status: "failed", message: "Uploaded file is empty." });
}
const results = [];
// Read CSV into memory (array of normalized rows)
const rows = [];
fs.createReadStream(filePath)
.pipe(csv())
.on("data", (row) => {
const cleanRow = {};
for (const key in row) {
cleanRow[key.trim()] = row[key]?.trim() || null;
.on("data", (rawRow) => {
// Normalize headers and map to canonical keys
const row = {};
for (const key in rawRow) {
const normalizedKey = key.replace(/[\s\W]+/g, "_").trim().toLowerCase();
// Standard canonical keys
if (normalizedKey === "establishment_id" || normalizedKey === "establishment_code" || normalizedKey.startsWith("establishment")) {
row["Establishment Id"] = rawRow[key]?.trim() || null;
continue;
}
if (normalizedKey.includes("factory name") || normalizedKey.includes("factory_name") || normalizedKey.includes("Factory Name")) {
row["Factory Name"] = rawRow[key]?.trim() || null;
continue;
}
if (normalizedKey.includes("Email") || normalizedKey.includes("email") ) {
row["Email"] = rawRow[key]?.trim() || null;
continue;
}
if (normalizedKey.includes("Emirate") || normalizedKey.includes("emirate") ) {
row["Emirate"] = rawRow[key]?.trim() || null;
continue;
}
if (normalizedKey.includes("total employment") || normalizedKey.includes("total_employment") || normalizedKey.includes("total employments")) {
row["Total Employment"] = rawRow[key]?.trim() || null;
continue;
}
// HS code columns: keep any header that contains 'hs' (hs_code, hs_code_1, hs1, etc.)
if (/^hs|hs_code|hs_code_\d|hs\d/i.test(normalizedKey) || normalizedKey.includes("hs")) {
// keep the exact normalized header so we can iterate later
row[normalizedKey] = rawRow[key]?.trim() || null;
continue;
}
// Fallback - keep other columns too
row[normalizedKey] = rawRow[key]?.trim() || null;
}
results.push(cleanRow);
rows.push(row);
})
.on("end", async () => {
try {
if (results.length === 0) {
removeFile(filePath);
if (!rows.length) {
if (fs.existsSync(filePath)) fs.unlinkSync(filePath);
return res.status(400).send({ status: "failed", message: "CSV file is empty or invalid." });
}
// Check required canonical columns exist in the header (based on first row keys)
const firstRowKeys = Object.keys(rows[0]);
const required = ["Establishment Id", "Factory Name", "Email", "Emirate", "Total Employment"];
const missing = required.filter((k) => !firstRowKeys.includes(k));
if (missing.length > 0) {
if (fs.existsSync(filePath)) fs.unlinkSync(filePath);
return res.status(400).send({
status: "failed",
message: "CSV file is empty or invalid.",
message: `Missing required columns: ${missing.join(", ")}`,
});
}
const requiredCols = [
"Establishment Id",
"Factory Name",
"Email",
"Emirate",
"Total Employment",
];
// Prepare maps: emirates and products
const emirates = await Emirate.findAll({ attributes: ["id", "name"] });
const emirateMap = {};
emirates.forEach((e) => {
if (e.name) emirateMap[e.name.trim().toLowerCase()] = e.id;
});
const headers = Object.keys(results[0]);
const missingCols = requiredCols.filter((col) => !headers.includes(col));
const extraCols = headers.filter(
(col) => !requiredCols.includes(col) && !col.toLowerCase().startsWith("hs code")
);
// Build product map by normalized hs_code (digits only, no leading zeros)
const products = await Product.findAll({ attributes: ["id", "hs_code"] });
const productMap = {};
products.forEach((p) => {
if (!p.hs_code) return;
const normalized = String(p.hs_code).replace(/[^0-9]/g, "").replace(/^0+/, "");
if (normalized) productMap[normalized] = p.id;
});
if (missingCols.length > 0 || extraCols.length > 0) {
removeFile(filePath);
let message = "";
if (missingCols.length > 0)
message += `Missing required columns: ${missingCols.join(", ")}. `;
if (extraCols.length > 0)
message += `Unexpected columns found: ${extraCols.join(", ")}. Only 'Establishment Id', 'Factory Name', 'Email', 'Emirate', 'Total Employment', and HS Code columns are allowed.`;
return res.status(400).send({
status: "failed",
message: message.trim(),
});
}
// Existing establishment codes in DB
const existingEsts = await Establishment.findAll({
attributes: ["establishment_code", "factory_name", "establishment_contact_email"]
});
const hsCodeCols = headers.filter((h) =>
h.toLowerCase().replace(/[\s_]+/g, "").startsWith("hscode")
);
const existingEstIdSet = new Set(existingEsts.map((e) => e.establishment_code));
const existingFactorySet = new Set(existingEsts.map((e) => e.factory_name?.trim().toLowerCase()));
const existingEmailSet = new Set(existingEsts.map((e) => e.establishment_contact_email?.trim().toLowerCase()));
const toInsert = [];
const duplicatesInFile = [];
const errors = [];
const seenKeys = new Set();
const seenEmails = new Set();
const fileDuplicates = new Set();
const seenEstIds = new Set();
const toInsert = [];
for (const [i, row] of results.entries()) {
const rowNum = i + 1;
const estId = row["Establishment Id"];
const factory = row["Factory Name"];
const email = row["Email"];
const emirate = row["Emirate"];
const emp = row["Total Employment"];
// Process each row: validate & prepare
for (let i = 0; i < rows.length; i++) {
const raw = rows[i];
const rowNumber = i + 1;
// Basic field check
if (!estId || !factory || !email || !emirate || !emp) {
errors.push({ row: rowNum, error: "Missing required fields." });
const estId = raw["Establishment Id"]?.trim();
const factory = raw["Factory Name"]?.trim();
const email = raw["Email"]?.trim();
const emirateRaw = raw["Emirate"]?.trim();
const employmentRaw = raw["Total Employment"]?.trim();
// Required fields
if (!estId || !factory || !email || !emirateRaw) {
errors.push({
row: rowNumber,
error: "Missing required fields (Establishment Id, Factory Name, Email, Emirate).",
});
continue;
}
// Email format
const emailRegex = /^[^\s@]+@[^\s@]+\.[^\s@]+$/;
if (!emailRegex.test(email)) {
errors.push({ row: rowNum, error: `Invalid email: ${email}` });
continue;
}
//Checking repeating data
const estIdRowMap = {};
const factoryRowMap = {};
const emailRowMap = {};
// Total Employment numeric
if (isNaN(emp)) {
errors.push({ row: rowNum, error: "Total Employment must be a number." });
continue;
}
// During row loop — replace your tracking code with this:
const keyEst = estId?.trim();
const keyFactory = factory?.trim().toLowerCase();
const keyEmail = email?.trim().toLowerCase();
// File-level duplicates
const key = `${estId}|${factory}|${email}|${emirate}`.toLowerCase();
if (seenKeys.has(key)) {
duplicatesInFile.push({ row: rowNum, error: "Duplicate record in file." });
continue;
}
seenKeys.add(key);
// Helper for inserting row numbers
const pushRow = (map, key) =>
key ? (map[key] = map[key] ? [...map[key], rowNumber] : [rowNumber]) : null;
if (seenEmails.has(email.toLowerCase())) {
duplicatesInFile.push({ row: rowNum, error: `Duplicate email in file: ${email}` });
continue;
}
seenEmails.add(email.toLowerCase());
// Track all 3
pushRow(estIdRowMap, keyEst);
pushRow(factoryRowMap, keyFactory);
pushRow(emailRowMap, keyEmail);
// Emirate validation
const emirateRecord = await Emirate.findOne({
where: Sequelize.where(
Sequelize.fn("LOWER", Sequelize.col("name")),
emirate.toLowerCase()
),
attributes: ["id"],
const duplicateDetails = [];
[
["Establishment Id", estIdRowMap],
["Factory Name", factoryRowMap],
["Email", emailRowMap],
].forEach(([field, map]) => {
Object.entries(map).forEach(([value, rows]) => {
if (rows.length > 1) {
duplicateDetails.push({
field,
value,
rows
});
}
});
if (!emirateRecord) {
errors.push({ row: rowNum, error: `Invalid Emirate: ${emirate}` });
});
if (duplicateDetails.length > 0) {
if (fs.existsSync(filePath)) fs.unlinkSync(filePath);
return res.status(400).send({
status: "failed",
message: "Repeated values found in uploaded file.",
duplicates: duplicateDetails,
});
}
const hsRawCodes = [];
for (const k of Object.keys(raw)) {
if (/^hs|hs_code|hs_code_\d|hs\d/i.test(k) || k.includes("hs")) {
const v = raw[k];
if (v && String(v).trim()) {
hsRawCodes.push(String(v).trim());
}
}
}
// Validation: at least one HS code must be present
if (hsRawCodes.length === 0) {
errors.push({
row: rowNumber,
error: "Missing HS Code columns or no HS Code values provided.",
});
continue;
}
// Extract and validate HS Codes
const hsCodes = hsCodeCols
.map((col) => row[col])
.filter(Boolean)
.flatMap((v) =>
v
.split(/[,\s]+/)
.map((x) => x.replace(/[-\/]/g, "").trim())
.filter((x) => /^\d+$/.test(x) && x.length === HS_CODE_LENGTH)
);
const seenEstIds = new Set();
const seenFactories = new Set();
const seenEmails = new Set();
// DB duplicate check
const existingEst = await Establishment.findOne({
where: {
[Op.or]: [
{ establishment_code: estId },
{ factory_name: factory },
{ establishment_contact_email: email },
],
},
attributes: ["establishment_code", "factory_name", "establishment_contact_email"],
if (seenEstIds.has(keyEst)) {
fileDuplicates.add(`Establishment ID: ${keyEst}`);
continue;
}
if (seenFactories.has(keyFactory)) {
fileDuplicates.add(`Factory Name: ${factory}`);
continue;
}
if (seenEmails.has(keyEmail)) {
fileDuplicates.add(`Email: ${email}`);
continue;
}
seenEstIds.add(keyEst);
seenFactories.add(keyFactory);
seenEmails.add(keyEmail);
if (existingEstIdSet.has(keyEst)) {
fileDuplicates.add(`Establishment ID already exists: ${keyEst}`);
continue;
}
if (existingFactorySet.has(keyFactory)) {
fileDuplicates.add(`Factory Name already exists: ${factory}`);
continue;
}
if (existingEmailSet.has(keyEmail)) {
fileDuplicates.add(`Email already exists: ${email}`);
continue;
}
// Emirate mapping
const emirateId = emirateMap[emirateRaw.toLowerCase()];
if (!emirateId) {
errors.push({
row: rowNumber,
error: `Invalid Emirate: '${emirateRaw}'`,
});
continue;
}
// Clean HS codes: remove non-digits and leading zeros
const cleanedHs = hsRawCodes
.map((hc) => hc.replace(/[^0-9]/g, "").replace(/^0+/, ""))
.filter(Boolean); // remove empty after cleaning
// Map to product IDs
const mappedProducts = cleanedHs
.map((c) => productMap[c])
.filter(Boolean);
// Validate mapping
if (mappedProducts.length === 0) {
errors.push({
row: rowNumber,
error: "None of the provided HS Codes match existing products.",
hs_codes: cleanedHs
});
continue;
}
if (mappedProducts.length === 0) {
errors.push({ row: rowNum, error: "No HS codes matched product list" });
continue;
}
// Validation: at least one HS must match existing product
if (mappedProducts.length === 0) {
errors.push({
row: rowNumber,
error: "None of the provided HS Codes match existing products.",
hs_codes: cleanedHs,
});
continue;
}
// parse employment
const employment = parseInt(employmentRaw) || 0;
toInsert.push({
estCode: estId,
factory,
email,
emirateId,
employment,
productIds: [...new Set(mappedProducts)],
rowIndex: rowNumber,
});
}
if (existingEst) {
const details = [];
if (existingEst.establishment_code === estId)
details.push({
field: "establishment_code",
message: "Establishment Id already exists",
});
if (existingEst.factory_name === factory)
details.push({
field: "factory_name",
message: "Factory Name already exists",
});
if (existingEst.establishment_contact_email === email)
details.push({
field: "establishment_contact_email",
message: "Email already exists",
});
// If file duplicates found (either in file or in DB), return error - consistent with product controller
if (fileDuplicates.size > 0) {
if (fs.existsSync(filePath)) fs.unlinkSync(filePath);
return res.status(400).send({
status: "failed",
message: "Duplicate Establishment IDs found (in file or already exist in system). Resolve and re-upload.",
duplicates: [...fileDuplicates],
});
}
removeFile(filePath);
return res.status(400).send({
status: "failed",
message: "Validation error",
details,
// Insert prepared establishments and associated products
let inserted = 0;
for (const rec of toInsert) {
try {
const createdEst = await Establishment.create({
establishment_code: rec.estCode,
factory_name: rec.factory,
establishment_contact_email: rec.email,
establishment_emirate_id: rec.emirateId,
total_employees: rec.employment,
created_by: req.user.id,
created_at: new Date(),
});
// prepare bulk create payload for establishment_products
const payload = rec.productIds.map((pid) => ({
establishment_id: createdEst.id,
product_id: pid,
created_by: req.user.id,
created_at: new Date(),
is_active: true,
}));
await EstablishmentProduct.bulkCreate(payload);
inserted++;
} catch (errInner) {
// Collect row-level DB insert errors for partial success reporting
errors.push({
row: rec.rowIndex,
error: `DB insert error: ${errInner.message}`,
});
}
// Passed validation
toInsert.push({
establishment_code: estId,
factory_name: factory,
establishment_contact_email: email,
establishment_emirate_id: emirateRecord.id,
total_employees: parseInt(emp),
hs_codes: hsCodes,
created_by: req.user.id,
created_at: new Date(),
});
}
// Stop if file duplicates found
if (duplicatesInFile.length > 0) {
removeFile(filePath);
return res.status(400).send({
status: "failed",
message: "Duplicate records found in file. Resolve and re-upload.",
duplicates: duplicatesInFile,
});
}
// cleanup uploaded file
if (fs.existsSync(filePath)) fs.unlinkSync(filePath);
const inserted = await Establishment.bulkCreate(toInsert, { validate: true });
removeFile(filePath);
let message = `${inserted.length} establishments inserted successfully.`;
// Build final response to match product import style
let finalStatus = "success";
let message = `${inserted} establishments inserted successfully.`;
if (errors.length > 0) {
finalStatus = inserted.length > 0 ? "partial_success" : "failed";
message = `${inserted.length} inserted, ${errors.length} skipped due to errors.`;
finalStatus = inserted > 0 ? "partial_success" : "failed";
if (finalStatus === "partial_success") {
message = `${inserted} establishments inserted, ${errors.length} rows failed.`;
} else {
message = `No establishments imported. ${errors.length} validation errors found.`;
}
}
return res.status(200).send({
status: finalStatus,
message,
summary: {
total_records: results.length,
imported: inserted.length,
errors: errors.length,
duplicates: duplicatesInFile.length,
total_records: rows.length,
imported: inserted,
errors,
},
errors,
});
} catch (err) {
removeFile(filePath);
console.error("Processing error:", err);
return res.status(500).send({
status: "failed",
message: err.message,
});
if (fs.existsSync(filePath)) fs.unlinkSync(filePath);
return res.status(500).send({ status: "failed", message: err.message });
}
})
.on("error", (err) => {
if (fs.existsSync(filePath)) fs.unlinkSync(filePath);
return res.status(500).send({ status: "failed", message: err.message });
});
} catch (error) {
console.error(error);
return res.status(500).send({
status: "failed",
message: error.message,
});
return res.status(500).send({ status: "failed", message: error.message });
}
};

View File

@ -25,7 +25,7 @@ module.exports = (sequelize, DataTypes) => {
establishment_contact_person_name: { type: DataTypes.STRING },
establishment_contact_person_designation: { type: DataTypes.STRING },
establishment_mobile_number: { type: DataTypes.STRING },
establishment_contact_email: { type: DataTypes.STRING },
establishment_contact_email: {type: DataTypes.STRING, allowNull: false, unique: true },
establishment_website: { type: DataTypes.STRING },
// Corporate/Head Office Contact Details