Column validation added

app/controllers/establishment.controller.js

@@ -559,8 +559,6 @@ exports.updateEstablishment = async (req, res) => {
 };
 
 
-
-
 // exports.updateEstablishment = async (req, res) => {
 //   try {
 
@@ -820,265 +818,392 @@ exports.forgotPasswordVerifyOTP = async (req, res) => {
 };
 
 exports.establishmentBulkUpload = async (req, res) => {
-  const removeFile = (path) => fs.existsSync(path) && fs.unlinkSync(path);
-  const HS_CODE_LENGTH = 10;
-
   try {
+    // Basic file & user validations
     if (!req.file) {
-      return res.status(400).send({ status: "failed", message: "No file uploaded." });
+      return res.status(400).send({ status: "failed", message: "No file uploaded" });
     }
-
     const filePath = req.file.path;
-    if (!req.file.originalname.endsWith(".csv")) {
+
-      removeFile(filePath);
+    if (!req.file.originalname.toLowerCase().endsWith(".csv")) {
-      return res.status(400).send({
+      if (fs.existsSync(filePath)) fs.unlinkSync(filePath);
-        status: "failed",
+      return res.status(400).send({ status: "failed", message: "Invalid file type. Only CSV allowed." });
-        message: "Invalid file type. Only CSV allowed.",
-      });
     }
 
-    const mode = req.body.mode?.toLowerCase() || "add";
+    if (!req.user?.id || isNaN(req.user.id)) {
+      if (fs.existsSync(filePath)) fs.unlinkSync(filePath);
+      return res.status(400).send({ status: "failed", message: "Invalid or missing User Id." });
+    }
+
+    const mode = (req.body.mode || "add").toLowerCase();
     if (mode !== "add") {
-      removeFile(filePath);
+      if (fs.existsSync(filePath)) fs.unlinkSync(filePath);
-      return res.status(400).send({
+      return res.status(400).send({ status: "failed", message: "Only 'Add Only' mode is supported currently." });
-        status: "failed",
-        message: "Only 'Add Only' mode is supported currently.",
-      });
     }
 
     const stats = fs.statSync(filePath);
     if (stats.size === 0) {
-      removeFile(filePath);
+      if (fs.existsSync(filePath)) fs.unlinkSync(filePath);
-      return res.status(400).send({
+      return res.status(400).send({ status: "failed", message: "Uploaded file is empty." });
-        status: "failed",
-        message: "Uploaded file is empty.",
-      });
     }
 
-    const results = [];
+    // Read CSV into memory (array of normalized rows)
-
+    const rows = [];
     fs.createReadStream(filePath)
       .pipe(csv())
-      .on("data", (row) => {
+      .on("data", (rawRow) => {
-        const cleanRow = {};
+        // Normalize headers and map to canonical keys
-        for (const key in row) {
+        const row = {};
-          cleanRow[key.trim()] = row[key]?.trim() || null;
+        for (const key in rawRow) {
+          const normalizedKey = key.replace(/[\s\W]+/g, "_").trim().toLowerCase();
+
+          // Standard canonical keys
+          if (normalizedKey === "establishment_id" || normalizedKey === "establishment_code" || normalizedKey.startsWith("establishment")) {
+            row["Establishment Id"] = rawRow[key]?.trim() || null;
+            continue;
+          }
+
+          if (normalizedKey.includes("factory name") || normalizedKey.includes("factory_name") || normalizedKey.includes("Factory Name")) {
+            row["Factory Name"] = rawRow[key]?.trim() || null;
+            continue;
+          }
+
+          if (normalizedKey.includes("Email") || normalizedKey.includes("email") ) {
+            row["Email"] = rawRow[key]?.trim() || null;
+            continue;
+          }
+
+          if (normalizedKey.includes("Emirate") || normalizedKey.includes("emirate") ) {
+            row["Emirate"] = rawRow[key]?.trim() || null;
+            continue;
+          }
+
+          if (normalizedKey.includes("total employment") || normalizedKey.includes("total_employment") || normalizedKey.includes("total employments")) {
+            row["Total Employment"] = rawRow[key]?.trim() || null;
+            continue;
+          }
+
+          // HS code columns: keep any header that contains 'hs' (hs_code, hs_code_1, hs1, etc.)
+          if (/^hs|hs_code|hs_code_\d|hs\d/i.test(normalizedKey) || normalizedKey.includes("hs")) {
+            // keep the exact normalized header so we can iterate later
+            row[normalizedKey] = rawRow[key]?.trim() || null;
+            continue;
+          }
+
+          // Fallback - keep other columns too
+          row[normalizedKey] = rawRow[key]?.trim() || null;
         }
-        results.push(cleanRow);
+
+        rows.push(row);
       })
       .on("end", async () => {
         try {
-          if (results.length === 0) {
+          if (!rows.length) {
-            removeFile(filePath);
+            if (fs.existsSync(filePath)) fs.unlinkSync(filePath);
+            return res.status(400).send({ status: "failed", message: "CSV file is empty or invalid." });
+          }
+
+          // Check required canonical columns exist in the header (based on first row keys)
+          const firstRowKeys = Object.keys(rows[0]);
+          const required = ["Establishment Id", "Factory Name", "Email", "Emirate", "Total Employment"];
+          const missing = required.filter((k) => !firstRowKeys.includes(k));
+          if (missing.length > 0) {
+            if (fs.existsSync(filePath)) fs.unlinkSync(filePath);
             return res.status(400).send({
               status: "failed",
-              message: "CSV file is empty or invalid.",
+              message: `Missing required columns: ${missing.join(", ")}`,
             });
           }
 
-          const requiredCols = [
+          // Prepare maps: emirates and products
-            "Establishment Id",
+          const emirates = await Emirate.findAll({ attributes: ["id", "name"] });
-            "Factory Name",
+          const emirateMap = {};
-            "Email",
+          emirates.forEach((e) => {
-            "Emirate",
+            if (e.name) emirateMap[e.name.trim().toLowerCase()] = e.id;
-            "Total Employment",
+          });
-          ];
 
-          const headers = Object.keys(results[0]);
+          // Build product map by normalized hs_code (digits only, no leading zeros)
-          const missingCols = requiredCols.filter((col) => !headers.includes(col));
+          const products = await Product.findAll({ attributes: ["id", "hs_code"] });
-          const extraCols = headers.filter(
+          const productMap = {};
-            (col) => !requiredCols.includes(col) && !col.toLowerCase().startsWith("hs code")
+          products.forEach((p) => {
-          );
+            if (!p.hs_code) return;
+            const normalized = String(p.hs_code).replace(/[^0-9]/g, "").replace(/^0+/, "");
+            if (normalized) productMap[normalized] = p.id;
+          });
 
-          if (missingCols.length > 0 || extraCols.length > 0) {
+          
-            removeFile(filePath);
+          // Existing establishment codes in DB
-            let message = "";
+          const existingEsts = await Establishment.findAll({
-            if (missingCols.length > 0)
+            attributes: ["establishment_code", "factory_name", "establishment_contact_email"]
-              message += `Missing required columns: ${missingCols.join(", ")}. `;
+          });
-            if (extraCols.length > 0)
-              message += `Unexpected columns found: ${extraCols.join(", ")}. Only 'Establishment Id', 'Factory Name', 'Email', 'Emirate', 'Total Employment', and HS Code columns are allowed.`;
-            return res.status(400).send({
-              status: "failed",
-              message: message.trim(),
-            });
-          }
 
-          const hsCodeCols = headers.filter((h) =>
+          const existingEstIdSet = new Set(existingEsts.map((e) => e.establishment_code));
-            h.toLowerCase().replace(/[\s_]+/g, "").startsWith("hscode")
+          const existingFactorySet = new Set(existingEsts.map((e) => e.factory_name?.trim().toLowerCase()));
-          );
+          const existingEmailSet = new Set(existingEsts.map((e) => e.establishment_contact_email?.trim().toLowerCase()));
 
-          const toInsert = [];
-          const duplicatesInFile = [];
           const errors = [];
-          const seenKeys = new Set();
+          const fileDuplicates = new Set();
-          const seenEmails = new Set();
+          const seenEstIds = new Set();
+          const toInsert = []; 
 
-          for (const [i, row] of results.entries()) {
+          // Process each row: validate & prepare
-            const rowNum = i + 1;
+          for (let i = 0; i < rows.length; i++) {
-            const estId = row["Establishment Id"];
+            const raw = rows[i];
-            const factory = row["Factory Name"];
+            const rowNumber = i + 1;
-            const email = row["Email"];
-            const emirate = row["Emirate"];
-            const emp = row["Total Employment"];
 
-            // Basic field check
+            const estId = raw["Establishment Id"]?.trim();
-            if (!estId || !factory || !email || !emirate || !emp) {
+            const factory = raw["Factory Name"]?.trim();
-              errors.push({ row: rowNum, error: "Missing required fields." });
+            const email = raw["Email"]?.trim();
+            const emirateRaw = raw["Emirate"]?.trim();
+            const employmentRaw = raw["Total Employment"]?.trim();
+
+            // Required fields
+            if (!estId || !factory || !email || !emirateRaw) {
+              errors.push({
+                row: rowNumber,
+                error: "Missing required fields (Establishment Id, Factory Name, Email, Emirate).",
+              });
               continue;
             }
 
-            // Email format
+            //Checking repeating data
-            const emailRegex = /^[^\s@]+@[^\s@]+\.[^\s@]+$/;
+          const estIdRowMap = {};
-            if (!emailRegex.test(email)) {
+          const factoryRowMap = {};
-              errors.push({ row: rowNum, error: `Invalid email: ${email}` });
+          const emailRowMap = {};
-              continue;
-            }
 
-            // Total Employment numeric
+          // During row loop — replace your tracking code with this:
-            if (isNaN(emp)) {
+          const keyEst = estId?.trim();
-              errors.push({ row: rowNum, error: "Total Employment must be a number." });
+          const keyFactory = factory?.trim().toLowerCase();
-              continue;
+          const keyEmail = email?.trim().toLowerCase();
-            }
 
-            // File-level duplicates
+          // Helper for inserting row numbers
-            const key = `${estId}|${factory}|${email}|${emirate}`.toLowerCase();
+          const pushRow = (map, key) =>
-            if (seenKeys.has(key)) {
+            key ? (map[key] = map[key] ? [...map[key], rowNumber] : [rowNumber]) : null;
-              duplicatesInFile.push({ row: rowNum, error: "Duplicate record in file." });
-              continue;
-            }
-            seenKeys.add(key);
 
-            if (seenEmails.has(email.toLowerCase())) {
+          // Track all 3
-              duplicatesInFile.push({ row: rowNum, error: `Duplicate email in file: ${email}` });
+          pushRow(estIdRowMap, keyEst);
-              continue;
+          pushRow(factoryRowMap, keyFactory);
-            }
+          pushRow(emailRowMap, keyEmail);
-            seenEmails.add(email.toLowerCase());
 
-            // Emirate validation
+          const duplicateDetails = [];
-            const emirateRecord = await Emirate.findOne({
+
-              where: Sequelize.where(
+          [
-                Sequelize.fn("LOWER", Sequelize.col("name")),
+            ["Establishment Id", estIdRowMap],
-                emirate.toLowerCase()
+            ["Factory Name", factoryRowMap],
-              ),
+            ["Email", emailRowMap],
-              attributes: ["id"],
+          ].forEach(([field, map]) => {
+            Object.entries(map).forEach(([value, rows]) => {
+              if (rows.length > 1) {
+                duplicateDetails.push({
+                  field,
+                  value,
+                  rows
+                });
+              }
             });
-            if (!emirateRecord) {
+          });
-              errors.push({ row: rowNum, error: `Invalid Emirate: ${emirate}` });
+          if (duplicateDetails.length > 0) {
+            if (fs.existsSync(filePath)) fs.unlinkSync(filePath);
+            return res.status(400).send({
+              status: "failed",
+              message: "Repeated values found in uploaded file.",
+              duplicates: duplicateDetails,
+            });
+          }
+          
+            const hsRawCodes = [];
+            for (const k of Object.keys(raw)) {
+              if (/^hs|hs_code|hs_code_\d|hs\d/i.test(k) || k.includes("hs")) {
+                const v = raw[k];
+                if (v && String(v).trim()) {
+                  hsRawCodes.push(String(v).trim());
+                }
+              }
+            }
+
+            // Validation: at least one HS code must be present
+            if (hsRawCodes.length === 0) {
+              errors.push({
+                row: rowNumber,
+                error: "Missing HS Code columns or no HS Code values provided.",
+              });
               continue;
             }
 
-            // Extract and validate HS Codes
+            const seenEstIds = new Set();
-            const hsCodes = hsCodeCols
+            const seenFactories = new Set();
-              .map((col) => row[col])
+            const seenEmails = new Set();
-              .filter(Boolean)
-              .flatMap((v) =>
-                v
-                  .split(/[,\s]+/)
-                  .map((x) => x.replace(/[-\/]/g, "").trim())
-                  .filter((x) => /^\d+$/.test(x) && x.length === HS_CODE_LENGTH)
-              );
 
-            // DB duplicate check
+            if (seenEstIds.has(keyEst)) {
-            const existingEst = await Establishment.findOne({
+              fileDuplicates.add(`Establishment ID: ${keyEst}`);
-              where: {
+              continue;
-                [Op.or]: [
+            }
-                  { establishment_code: estId },
+            if (seenFactories.has(keyFactory)) {
-                  { factory_name: factory },
+              fileDuplicates.add(`Factory Name: ${factory}`);
-                  { establishment_contact_email: email },
+              continue;
-                ],
+            }
-              },
+            if (seenEmails.has(keyEmail)) {
-              attributes: ["establishment_code", "factory_name", "establishment_contact_email"],
+              fileDuplicates.add(`Email: ${email}`);
+              continue;
+            }
+
+            seenEstIds.add(keyEst);
+            seenFactories.add(keyFactory);
+            seenEmails.add(keyEmail);
+
+            if (existingEstIdSet.has(keyEst)) {
+              fileDuplicates.add(`Establishment ID already exists: ${keyEst}`);
+              continue;
+            }
+            if (existingFactorySet.has(keyFactory)) {
+              fileDuplicates.add(`Factory Name already exists: ${factory}`);
+              continue;
+            }
+            if (existingEmailSet.has(keyEmail)) {
+              fileDuplicates.add(`Email already exists: ${email}`);
+              continue;
+            }
+
+            // Emirate mapping
+            const emirateId = emirateMap[emirateRaw.toLowerCase()];
+            if (!emirateId) {
+              errors.push({
+                row: rowNumber,
+                error: `Invalid Emirate: '${emirateRaw}'`,
+              });
+              continue;
+            }
+
+            // Clean HS codes: remove non-digits and leading zeros
+            const cleanedHs = hsRawCodes
+              .map((hc) => hc.replace(/[^0-9]/g, "").replace(/^0+/, ""))
+              .filter(Boolean); // remove empty after cleaning
+
+            // Map to product IDs
+            const mappedProducts = cleanedHs
+              .map((c) => productMap[c])
+              .filter(Boolean);
+
+            // Validate mapping
+            if (mappedProducts.length === 0) {
+              errors.push({
+                row: rowNumber,
+                error: "None of the provided HS Codes match existing products.",
+                hs_codes: cleanedHs
+              });
+              continue;
+            }                  
+
+            if (mappedProducts.length === 0) {
+              errors.push({ row: rowNum, error: "No HS codes matched product list" });
+              continue;
+            }
+            // Validation: at least one HS must match existing product
+            if (mappedProducts.length === 0) {
+              errors.push({
+                row: rowNumber,
+                error: "None of the provided HS Codes match existing products.",
+                hs_codes: cleanedHs,
+              });
+              continue;
+            }
+           
+            // parse employment
+            const employment = parseInt(employmentRaw) || 0;
+
+            toInsert.push({
+              estCode: estId,
+              factory,
+              email,
+              emirateId,
+              employment,
+              productIds: [...new Set(mappedProducts)],
+              rowIndex: rowNumber,
             });
+          } 
 
-            if (existingEst) {
+          // If file duplicates found (either in file or in DB), return error - consistent with product controller
-              const details = [];
+          if (fileDuplicates.size > 0) {
-              if (existingEst.establishment_code === estId)
+            if (fs.existsSync(filePath)) fs.unlinkSync(filePath);
-                details.push({
+            return res.status(400).send({
-                  field: "establishment_code",
+              status: "failed",
-                  message: "Establishment Id already exists",
+              message: "Duplicate Establishment IDs found (in file or already exist in system). Resolve and re-upload.",
-                });
+              duplicates: [...fileDuplicates],
-              if (existingEst.factory_name === factory)
+            });
-                details.push({
+          }
-                  field: "factory_name",
-                  message: "Factory Name already exists",
-                });
-              if (existingEst.establishment_contact_email === email)
-                details.push({
-                  field: "establishment_contact_email",
-                  message: "Email already exists",
-                });
 
-              removeFile(filePath);
+          // Insert prepared establishments and associated products
-              return res.status(400).send({
+          let inserted = 0;
-                status: "failed",
+          for (const rec of toInsert) {
-                message: "Validation error",
+            try {
-                details,
+              const createdEst = await Establishment.create({
+                establishment_code: rec.estCode,
+                factory_name: rec.factory,
+                establishment_contact_email: rec.email,
+                establishment_emirate_id: rec.emirateId,
+                total_employees: rec.employment,
+                created_by: req.user.id,
+                created_at: new Date(),
+              });
+
+              // prepare bulk create payload for establishment_products
+              const payload = rec.productIds.map((pid) => ({
+                establishment_id: createdEst.id, 
+                product_id: pid,
+                created_by: req.user.id,
+                created_at: new Date(),
+                is_active: true,
+              }));
+
+              await EstablishmentProduct.bulkCreate(payload);
+              inserted++;
+            } catch (errInner) {
+              // Collect row-level DB insert errors for partial success reporting
+              errors.push({
+                row: rec.rowIndex,
+                error: `DB insert error: ${errInner.message}`,
               });
             }
-
-            // Passed validation
-            toInsert.push({
-              establishment_code: estId,
-              factory_name: factory,
-              establishment_contact_email: email,
-              establishment_emirate_id: emirateRecord.id,
-              total_employees: parseInt(emp),
-              hs_codes: hsCodes,
-              created_by: req.user.id,
-              created_at: new Date(),
-            });
           }
 
-          // Stop if file duplicates found
+          // cleanup uploaded file
-          if (duplicatesInFile.length > 0) {
+          if (fs.existsSync(filePath)) fs.unlinkSync(filePath);
-            removeFile(filePath);
-            return res.status(400).send({
-              status: "failed",
-              message: "Duplicate records found in file. Resolve and re-upload.",
-              duplicates: duplicatesInFile,
-            });
-          }
 
-          const inserted = await Establishment.bulkCreate(toInsert, { validate: true });
+          // Build final response to match product import style
-
-          removeFile(filePath);
-
-          let message = `${inserted.length} establishments inserted successfully.`;
           let finalStatus = "success";
+          let message = `${inserted} establishments inserted successfully.`;
+
           if (errors.length > 0) {
-            finalStatus = inserted.length > 0 ? "partial_success" : "failed";
+            finalStatus = inserted > 0 ? "partial_success" : "failed";
-            message = `${inserted.length} inserted, ${errors.length} skipped due to errors.`;
+            if (finalStatus === "partial_success") {
+              message = `${inserted} establishments inserted, ${errors.length} rows failed.`;
+            } else {
+              message = `No establishments imported. ${errors.length} validation errors found.`;
+            }
           }
 
           return res.status(200).send({
             status: finalStatus,
             message,
             summary: {
-              total_records: results.length,
+              total_records: rows.length,
-              imported: inserted.length,
+              imported: inserted,
-              errors: errors.length,
+              errors,
-              duplicates: duplicatesInFile.length,
             },
-            errors,
           });
         } catch (err) {
-          removeFile(filePath);
+          if (fs.existsSync(filePath)) fs.unlinkSync(filePath);
-          console.error("Processing error:", err);
+          return res.status(500).send({ status: "failed", message: err.message });
-          return res.status(500).send({
-            status: "failed",
-            message: err.message,
-          });
         }
+      })
+      .on("error", (err) => {
+        if (fs.existsSync(filePath)) fs.unlinkSync(filePath);
+        return res.status(500).send({ status: "failed", message: err.message });
       });
   } catch (error) {
-    console.error(error);
+    return res.status(500).send({ status: "failed", message: error.message });
-    return res.status(500).send({
-      status: "failed",
-      message: error.message,
-    });
   }
 };
 
 
 
-

app/models/establishment.model.js

@@ -25,7 +25,7 @@ module.exports = (sequelize, DataTypes) => {
         establishment_contact_person_name: { type: DataTypes.STRING },
         establishment_contact_person_designation: { type: DataTypes.STRING },
         establishment_mobile_number: { type: DataTypes.STRING },
-        establishment_contact_email: { type: DataTypes.STRING },
+       establishment_contact_email: {type: DataTypes.STRING, allowNull: false, unique: true },
         establishment_website: { type: DataTypes.STRING },
 
         // Corporate/Head Office Contact Details