diff --git a/app/controllers/establishment.controller.js b/app/controllers/establishment.controller.js index 8371aa6..28cd8e0 100644 --- a/app/controllers/establishment.controller.js +++ b/app/controllers/establishment.controller.js @@ -559,8 +559,6 @@ exports.updateEstablishment = async (req, res) => { }; - - // exports.updateEstablishment = async (req, res) => { // try { @@ -820,265 +818,392 @@ exports.forgotPasswordVerifyOTP = async (req, res) => { }; exports.establishmentBulkUpload = async (req, res) => { - const removeFile = (path) => fs.existsSync(path) && fs.unlinkSync(path); - const HS_CODE_LENGTH = 10; - try { + // Basic file & user validations if (!req.file) { - return res.status(400).send({ status: "failed", message: "No file uploaded." }); + return res.status(400).send({ status: "failed", message: "No file uploaded" }); } - const filePath = req.file.path; - if (!req.file.originalname.endsWith(".csv")) { - removeFile(filePath); - return res.status(400).send({ - status: "failed", - message: "Invalid file type. Only CSV allowed.", - }); + + if (!req.file.originalname.toLowerCase().endsWith(".csv")) { + if (fs.existsSync(filePath)) fs.unlinkSync(filePath); + return res.status(400).send({ status: "failed", message: "Invalid file type. Only CSV allowed." }); } - const mode = req.body.mode?.toLowerCase() || "add"; + if (!req.user?.id || isNaN(req.user.id)) { + if (fs.existsSync(filePath)) fs.unlinkSync(filePath); + return res.status(400).send({ status: "failed", message: "Invalid or missing User Id." }); + } + + const mode = (req.body.mode || "add").toLowerCase(); if (mode !== "add") { - removeFile(filePath); - return res.status(400).send({ - status: "failed", - message: "Only 'Add Only' mode is supported currently.", - }); + if (fs.existsSync(filePath)) fs.unlinkSync(filePath); + return res.status(400).send({ status: "failed", message: "Only 'Add Only' mode is supported currently." }); } const stats = fs.statSync(filePath); if (stats.size === 0) { - removeFile(filePath); - return res.status(400).send({ - status: "failed", - message: "Uploaded file is empty.", - }); + if (fs.existsSync(filePath)) fs.unlinkSync(filePath); + return res.status(400).send({ status: "failed", message: "Uploaded file is empty." }); } - const results = []; - + // Read CSV into memory (array of normalized rows) + const rows = []; fs.createReadStream(filePath) .pipe(csv()) - .on("data", (row) => { - const cleanRow = {}; - for (const key in row) { - cleanRow[key.trim()] = row[key]?.trim() || null; + .on("data", (rawRow) => { + // Normalize headers and map to canonical keys + const row = {}; + for (const key in rawRow) { + const normalizedKey = key.replace(/[\s\W]+/g, "_").trim().toLowerCase(); + + // Standard canonical keys + if (normalizedKey === "establishment_id" || normalizedKey === "establishment_code" || normalizedKey.startsWith("establishment")) { + row["Establishment Id"] = rawRow[key]?.trim() || null; + continue; + } + + if (normalizedKey.includes("factory name") || normalizedKey.includes("factory_name") || normalizedKey.includes("Factory Name")) { + row["Factory Name"] = rawRow[key]?.trim() || null; + continue; + } + + if (normalizedKey.includes("Email") || normalizedKey.includes("email") ) { + row["Email"] = rawRow[key]?.trim() || null; + continue; + } + + if (normalizedKey.includes("Emirate") || normalizedKey.includes("emirate") ) { + row["Emirate"] = rawRow[key]?.trim() || null; + continue; + } + + if (normalizedKey.includes("total employment") || normalizedKey.includes("total_employment") || normalizedKey.includes("total employments")) { + row["Total Employment"] = rawRow[key]?.trim() || null; + continue; + } + + // HS code columns: keep any header that contains 'hs' (hs_code, hs_code_1, hs1, etc.) + if (/^hs|hs_code|hs_code_\d|hs\d/i.test(normalizedKey) || normalizedKey.includes("hs")) { + // keep the exact normalized header so we can iterate later + row[normalizedKey] = rawRow[key]?.trim() || null; + continue; + } + + // Fallback - keep other columns too + row[normalizedKey] = rawRow[key]?.trim() || null; } - results.push(cleanRow); + + rows.push(row); }) .on("end", async () => { try { - if (results.length === 0) { - removeFile(filePath); + if (!rows.length) { + if (fs.existsSync(filePath)) fs.unlinkSync(filePath); + return res.status(400).send({ status: "failed", message: "CSV file is empty or invalid." }); + } + + // Check required canonical columns exist in the header (based on first row keys) + const firstRowKeys = Object.keys(rows[0]); + const required = ["Establishment Id", "Factory Name", "Email", "Emirate", "Total Employment"]; + const missing = required.filter((k) => !firstRowKeys.includes(k)); + if (missing.length > 0) { + if (fs.existsSync(filePath)) fs.unlinkSync(filePath); return res.status(400).send({ status: "failed", - message: "CSV file is empty or invalid.", + message: `Missing required columns: ${missing.join(", ")}`, }); } - const requiredCols = [ - "Establishment Id", - "Factory Name", - "Email", - "Emirate", - "Total Employment", - ]; + // Prepare maps: emirates and products + const emirates = await Emirate.findAll({ attributes: ["id", "name"] }); + const emirateMap = {}; + emirates.forEach((e) => { + if (e.name) emirateMap[e.name.trim().toLowerCase()] = e.id; + }); - const headers = Object.keys(results[0]); - const missingCols = requiredCols.filter((col) => !headers.includes(col)); - const extraCols = headers.filter( - (col) => !requiredCols.includes(col) && !col.toLowerCase().startsWith("hs code") - ); + // Build product map by normalized hs_code (digits only, no leading zeros) + const products = await Product.findAll({ attributes: ["id", "hs_code"] }); + const productMap = {}; + products.forEach((p) => { + if (!p.hs_code) return; + const normalized = String(p.hs_code).replace(/[^0-9]/g, "").replace(/^0+/, ""); + if (normalized) productMap[normalized] = p.id; + }); - if (missingCols.length > 0 || extraCols.length > 0) { - removeFile(filePath); - let message = ""; - if (missingCols.length > 0) - message += `Missing required columns: ${missingCols.join(", ")}. `; - if (extraCols.length > 0) - message += `Unexpected columns found: ${extraCols.join(", ")}. Only 'Establishment Id', 'Factory Name', 'Email', 'Emirate', 'Total Employment', and HS Code columns are allowed.`; - return res.status(400).send({ - status: "failed", - message: message.trim(), - }); - } + + // Existing establishment codes in DB + const existingEsts = await Establishment.findAll({ + attributes: ["establishment_code", "factory_name", "establishment_contact_email"] + }); - const hsCodeCols = headers.filter((h) => - h.toLowerCase().replace(/[\s_]+/g, "").startsWith("hscode") - ); + const existingEstIdSet = new Set(existingEsts.map((e) => e.establishment_code)); + const existingFactorySet = new Set(existingEsts.map((e) => e.factory_name?.trim().toLowerCase())); + const existingEmailSet = new Set(existingEsts.map((e) => e.establishment_contact_email?.trim().toLowerCase())); - const toInsert = []; - const duplicatesInFile = []; const errors = []; - const seenKeys = new Set(); - const seenEmails = new Set(); + const fileDuplicates = new Set(); + const seenEstIds = new Set(); + const toInsert = []; - for (const [i, row] of results.entries()) { - const rowNum = i + 1; - const estId = row["Establishment Id"]; - const factory = row["Factory Name"]; - const email = row["Email"]; - const emirate = row["Emirate"]; - const emp = row["Total Employment"]; + // Process each row: validate & prepare + for (let i = 0; i < rows.length; i++) { + const raw = rows[i]; + const rowNumber = i + 1; - // Basic field check - if (!estId || !factory || !email || !emirate || !emp) { - errors.push({ row: rowNum, error: "Missing required fields." }); + const estId = raw["Establishment Id"]?.trim(); + const factory = raw["Factory Name"]?.trim(); + const email = raw["Email"]?.trim(); + const emirateRaw = raw["Emirate"]?.trim(); + const employmentRaw = raw["Total Employment"]?.trim(); + + // Required fields + if (!estId || !factory || !email || !emirateRaw) { + errors.push({ + row: rowNumber, + error: "Missing required fields (Establishment Id, Factory Name, Email, Emirate).", + }); continue; } - // Email format - const emailRegex = /^[^\s@]+@[^\s@]+\.[^\s@]+$/; - if (!emailRegex.test(email)) { - errors.push({ row: rowNum, error: `Invalid email: ${email}` }); - continue; - } + //Checking repeating data + const estIdRowMap = {}; + const factoryRowMap = {}; + const emailRowMap = {}; - // Total Employment numeric - if (isNaN(emp)) { - errors.push({ row: rowNum, error: "Total Employment must be a number." }); - continue; - } + // During row loop — replace your tracking code with this: + const keyEst = estId?.trim(); + const keyFactory = factory?.trim().toLowerCase(); + const keyEmail = email?.trim().toLowerCase(); - // File-level duplicates - const key = `${estId}|${factory}|${email}|${emirate}`.toLowerCase(); - if (seenKeys.has(key)) { - duplicatesInFile.push({ row: rowNum, error: "Duplicate record in file." }); - continue; - } - seenKeys.add(key); + // Helper for inserting row numbers + const pushRow = (map, key) => + key ? (map[key] = map[key] ? [...map[key], rowNumber] : [rowNumber]) : null; - if (seenEmails.has(email.toLowerCase())) { - duplicatesInFile.push({ row: rowNum, error: `Duplicate email in file: ${email}` }); - continue; - } - seenEmails.add(email.toLowerCase()); + // Track all 3 + pushRow(estIdRowMap, keyEst); + pushRow(factoryRowMap, keyFactory); + pushRow(emailRowMap, keyEmail); - // Emirate validation - const emirateRecord = await Emirate.findOne({ - where: Sequelize.where( - Sequelize.fn("LOWER", Sequelize.col("name")), - emirate.toLowerCase() - ), - attributes: ["id"], + const duplicateDetails = []; + + [ + ["Establishment Id", estIdRowMap], + ["Factory Name", factoryRowMap], + ["Email", emailRowMap], + ].forEach(([field, map]) => { + Object.entries(map).forEach(([value, rows]) => { + if (rows.length > 1) { + duplicateDetails.push({ + field, + value, + rows + }); + } }); - if (!emirateRecord) { - errors.push({ row: rowNum, error: `Invalid Emirate: ${emirate}` }); + }); + if (duplicateDetails.length > 0) { + if (fs.existsSync(filePath)) fs.unlinkSync(filePath); + return res.status(400).send({ + status: "failed", + message: "Repeated values found in uploaded file.", + duplicates: duplicateDetails, + }); + } + + const hsRawCodes = []; + for (const k of Object.keys(raw)) { + if (/^hs|hs_code|hs_code_\d|hs\d/i.test(k) || k.includes("hs")) { + const v = raw[k]; + if (v && String(v).trim()) { + hsRawCodes.push(String(v).trim()); + } + } + } + + // Validation: at least one HS code must be present + if (hsRawCodes.length === 0) { + errors.push({ + row: rowNumber, + error: "Missing HS Code columns or no HS Code values provided.", + }); continue; } - // Extract and validate HS Codes - const hsCodes = hsCodeCols - .map((col) => row[col]) - .filter(Boolean) - .flatMap((v) => - v - .split(/[,\s]+/) - .map((x) => x.replace(/[-\/]/g, "").trim()) - .filter((x) => /^\d+$/.test(x) && x.length === HS_CODE_LENGTH) - ); + const seenEstIds = new Set(); + const seenFactories = new Set(); + const seenEmails = new Set(); - // DB duplicate check - const existingEst = await Establishment.findOne({ - where: { - [Op.or]: [ - { establishment_code: estId }, - { factory_name: factory }, - { establishment_contact_email: email }, - ], - }, - attributes: ["establishment_code", "factory_name", "establishment_contact_email"], + if (seenEstIds.has(keyEst)) { + fileDuplicates.add(`Establishment ID: ${keyEst}`); + continue; + } + if (seenFactories.has(keyFactory)) { + fileDuplicates.add(`Factory Name: ${factory}`); + continue; + } + if (seenEmails.has(keyEmail)) { + fileDuplicates.add(`Email: ${email}`); + continue; + } + + seenEstIds.add(keyEst); + seenFactories.add(keyFactory); + seenEmails.add(keyEmail); + + if (existingEstIdSet.has(keyEst)) { + fileDuplicates.add(`Establishment ID already exists: ${keyEst}`); + continue; + } + if (existingFactorySet.has(keyFactory)) { + fileDuplicates.add(`Factory Name already exists: ${factory}`); + continue; + } + if (existingEmailSet.has(keyEmail)) { + fileDuplicates.add(`Email already exists: ${email}`); + continue; + } + + // Emirate mapping + const emirateId = emirateMap[emirateRaw.toLowerCase()]; + if (!emirateId) { + errors.push({ + row: rowNumber, + error: `Invalid Emirate: '${emirateRaw}'`, + }); + continue; + } + + // Clean HS codes: remove non-digits and leading zeros + const cleanedHs = hsRawCodes + .map((hc) => hc.replace(/[^0-9]/g, "").replace(/^0+/, "")) + .filter(Boolean); // remove empty after cleaning + + // Map to product IDs + const mappedProducts = cleanedHs + .map((c) => productMap[c]) + .filter(Boolean); + + // Validate mapping + if (mappedProducts.length === 0) { + errors.push({ + row: rowNumber, + error: "None of the provided HS Codes match existing products.", + hs_codes: cleanedHs + }); + continue; + } + + if (mappedProducts.length === 0) { + errors.push({ row: rowNum, error: "No HS codes matched product list" }); + continue; + } + // Validation: at least one HS must match existing product + if (mappedProducts.length === 0) { + errors.push({ + row: rowNumber, + error: "None of the provided HS Codes match existing products.", + hs_codes: cleanedHs, + }); + continue; + } + + // parse employment + const employment = parseInt(employmentRaw) || 0; + + toInsert.push({ + estCode: estId, + factory, + email, + emirateId, + employment, + productIds: [...new Set(mappedProducts)], + rowIndex: rowNumber, }); + } - if (existingEst) { - const details = []; - if (existingEst.establishment_code === estId) - details.push({ - field: "establishment_code", - message: "Establishment Id already exists", - }); - if (existingEst.factory_name === factory) - details.push({ - field: "factory_name", - message: "Factory Name already exists", - }); - if (existingEst.establishment_contact_email === email) - details.push({ - field: "establishment_contact_email", - message: "Email already exists", - }); + // If file duplicates found (either in file or in DB), return error - consistent with product controller + if (fileDuplicates.size > 0) { + if (fs.existsSync(filePath)) fs.unlinkSync(filePath); + return res.status(400).send({ + status: "failed", + message: "Duplicate Establishment IDs found (in file or already exist in system). Resolve and re-upload.", + duplicates: [...fileDuplicates], + }); + } - removeFile(filePath); - return res.status(400).send({ - status: "failed", - message: "Validation error", - details, + // Insert prepared establishments and associated products + let inserted = 0; + for (const rec of toInsert) { + try { + const createdEst = await Establishment.create({ + establishment_code: rec.estCode, + factory_name: rec.factory, + establishment_contact_email: rec.email, + establishment_emirate_id: rec.emirateId, + total_employees: rec.employment, + created_by: req.user.id, + created_at: new Date(), + }); + + // prepare bulk create payload for establishment_products + const payload = rec.productIds.map((pid) => ({ + establishment_id: createdEst.id, + product_id: pid, + created_by: req.user.id, + created_at: new Date(), + is_active: true, + })); + + await EstablishmentProduct.bulkCreate(payload); + inserted++; + } catch (errInner) { + // Collect row-level DB insert errors for partial success reporting + errors.push({ + row: rec.rowIndex, + error: `DB insert error: ${errInner.message}`, }); } - - // Passed validation - toInsert.push({ - establishment_code: estId, - factory_name: factory, - establishment_contact_email: email, - establishment_emirate_id: emirateRecord.id, - total_employees: parseInt(emp), - hs_codes: hsCodes, - created_by: req.user.id, - created_at: new Date(), - }); } - // Stop if file duplicates found - if (duplicatesInFile.length > 0) { - removeFile(filePath); - return res.status(400).send({ - status: "failed", - message: "Duplicate records found in file. Resolve and re-upload.", - duplicates: duplicatesInFile, - }); - } + // cleanup uploaded file + if (fs.existsSync(filePath)) fs.unlinkSync(filePath); - const inserted = await Establishment.bulkCreate(toInsert, { validate: true }); - - removeFile(filePath); - - let message = `${inserted.length} establishments inserted successfully.`; + // Build final response to match product import style let finalStatus = "success"; + let message = `${inserted} establishments inserted successfully.`; + if (errors.length > 0) { - finalStatus = inserted.length > 0 ? "partial_success" : "failed"; - message = `${inserted.length} inserted, ${errors.length} skipped due to errors.`; + finalStatus = inserted > 0 ? "partial_success" : "failed"; + if (finalStatus === "partial_success") { + message = `${inserted} establishments inserted, ${errors.length} rows failed.`; + } else { + message = `No establishments imported. ${errors.length} validation errors found.`; + } } return res.status(200).send({ status: finalStatus, message, summary: { - total_records: results.length, - imported: inserted.length, - errors: errors.length, - duplicates: duplicatesInFile.length, + total_records: rows.length, + imported: inserted, + errors, }, - errors, }); } catch (err) { - removeFile(filePath); - console.error("Processing error:", err); - return res.status(500).send({ - status: "failed", - message: err.message, - }); + if (fs.existsSync(filePath)) fs.unlinkSync(filePath); + return res.status(500).send({ status: "failed", message: err.message }); } + }) + .on("error", (err) => { + if (fs.existsSync(filePath)) fs.unlinkSync(filePath); + return res.status(500).send({ status: "failed", message: err.message }); }); } catch (error) { - console.error(error); - return res.status(500).send({ - status: "failed", - message: error.message, - }); + return res.status(500).send({ status: "failed", message: error.message }); } }; - diff --git a/app/models/establishment.model.js b/app/models/establishment.model.js index 2526b7e..96e877e 100644 --- a/app/models/establishment.model.js +++ b/app/models/establishment.model.js @@ -25,7 +25,7 @@ module.exports = (sequelize, DataTypes) => { establishment_contact_person_name: { type: DataTypes.STRING }, establishment_contact_person_designation: { type: DataTypes.STRING }, establishment_mobile_number: { type: DataTypes.STRING }, - establishment_contact_email: { type: DataTypes.STRING }, + establishment_contact_email: {type: DataTypes.STRING, allowNull: false, unique: true }, establishment_website: { type: DataTypes.STRING }, // Corporate/Head Office Contact Details