diff --git a/app/controllers/establishment.controller.js b/app/controllers/establishment.controller.js index 5813768..56ae38d 100644 --- a/app/controllers/establishment.controller.js +++ b/app/controllers/establishment.controller.js @@ -1107,7 +1107,7 @@ exports.establishmentBulkUpload = async (req, res) => { const rows = []; // ------------------------------------------------------- - // READ CSV AND SKIP EMPTY ROWS + // READ CSV AND NORMALIZE HEADERS // ------------------------------------------------------- await new Promise((resolve, reject) => { fs.createReadStream(filePath) @@ -1115,109 +1115,87 @@ exports.establishmentBulkUpload = async (req, res) => { .on("data", (rawRow) => { const row = {}; - // Normalize headers for (const key in rawRow) { - const normalizedKey = key.replace(/[\s\W]+/g, "_") + const normalizedKey = key + .replace(/[\s\W]+/g, "_") .trim() .toLowerCase(); row[normalizedKey] = rawRow[key]?.trim() || ""; } - const cleanedValues = Object.values(row).map(v =>(v || "").replace(/\s+/g, "").trim()); - const isEmpty = cleanedValues.every(v => v === ""); - if (isEmpty) return; - - rows.push(row); + const cleaned = Object.values(row).map(v => (v || "").replace(/\s+/g, "").trim()); + const empty = cleaned.every(v => v === ""); + if (!empty) rows.push(row); }) .on("end", resolve) .on("error", reject); }); - if (rows.length === 0) { - fs.unlinkSync(filePath); - return res.status(400).send({ - status: "failed", - message: "CSV file is empty.", - errors: [{ error: "Uploaded file is empty." }] - }); - } - // ------------------------------------------------------- - // REQUIRED HEADER CHECK - // ------------------------------------------------------- - const required = [ - "establishment_id", - "factory_name", - "email", - "emirate", - "total_employment" - ]; - - const firstRowKeys = Object.keys(rows[0]); - - const missing = required.filter(r => !firstRowKeys.includes(r)); - if (missing.length > 0) { + if (rows.length === 0) { fs.unlinkSync(filePath); return res.status(400).send({ status: "failed", - message: "Invalid data. Please check the instructions given and upload again.", - errors: [ - { error: "Missing required columns: " + missing.join(", ") } - ] + message: "CSV file is empty." }); } // ------------------------------------------------------- - // LOAD REFERENCE TABLES + // REQUIRED HEADERS + // ------------------------------------------------------- + const required = ["establishment_id", "factory_name", "user_name", "email", "emirate"]; + const firstKeys = Object.keys(rows[0]); + + const missingHeaders = required.filter(h => !firstKeys.includes(h)); + if (missingHeaders.length > 0) { + fs.unlinkSync(filePath); + return res.status(400).send({ + status: "failed", + message: "Missing required columns: " + missingHeaders.join(", ") + }); + } + + // ------------------------------------------------------- + // LOAD REFERENCE TABLES ONCE // ------------------------------------------------------- const emirates = await Emirate.findAll({ attributes: ["id", "name"] }); const emirateMap = {}; - emirates.forEach(e => { - emirateMap[e.name.trim().toLowerCase()] = e.id; - }); + emirates.forEach(e => (emirateMap[e.name.trim().toLowerCase()] = e.id)); - function normalizeHS(val) { + const cityTowns = await CityTown.findAll({ attributes: ["id", "name"] }); + const cityTownMap = {}; + cityTowns.forEach(ct => (cityTownMap[ct.name.trim().toLowerCase()] = ct.id)); + + function normalizeHS(val) { if (!val) return ""; - - return val - .toString() - .normalize("NFKD") - .replace(/[^\d]/g, "") - .trim(); + return val.toString().normalize("NFKD").replace(/[^\d]/g, "").trim(); } const products = await Product.findAll({ attributes: ["id", "hs_code"] }); - const productMap = {}; + products.forEach(p => { + const c = normalizeHS(p.hs_code); + if (c.length >= 1 && c.length <= 10) productMap[c] = p.id; + }); - products.forEach(p => { - const cleaned = normalizeHS(p.hs_code); - - // Only store valid HS codes 1–10 digits - if (cleaned.length >= 1 && cleaned.length <= 10) { - productMap[cleaned] = p.id; - } - }); - const existingEsts = await Establishment.findAll({ + const existing = await Establishment.findAll({ attributes: ["establishment_code", "factory_name", "establishment_contact_email"] }); - const existingEstIdSet = new Set(existingEsts.map(e => e.establishment_code)); - const existingFactorySet = new Set(existingEsts.map(e => e.factory_name.toLowerCase())); - const existingEmailSet = new Set(existingEsts.map(e => e.establishment_contact_email.toLowerCase())); + const existingEstSet = new Set(existing.map(e => e.establishment_code)); + const existingFactorySet = new Set(existing.map(e => e.factory_name.toLowerCase())); + const existingEmailSet = new Set(existing.map(e => e.establishment_contact_email.toLowerCase())); - // ------------------------------------------------------- - // FILE-LEVEL DUPLICATE TRACKING - // ------------------------------------------------------- - const errors = []; + // FILE duplicate trackers const fileEstSet = new Set(); const fileFactorySet = new Set(); const fileEmailSet = new Set(); + const errors = []; const prepared = []; // ------------------------------------------------------- - // VALIDATE EACH ROW + // PHASE 1: VALIDATE ALL ROWS (NO DB INSERT HERE) // ------------------------------------------------------- for (let i = 0; i < rows.length; i++) { const r = rows[i]; @@ -1227,42 +1205,24 @@ exports.establishmentBulkUpload = async (req, res) => { const factory = r.factory_name; const email = r.email; const emirate = r.emirate; - const employment = r.total_employment; - // Required validation - const missingFields = []; + const missing = []; + if (!est) missing.push("Establishment ID"); + if (!factory) missing.push("Factory Name"); + if (!r.user_name) missing.push("User Name"); + if (!email) missing.push("Email"); + if (!emirate) missing.push("Emirate"); - if (!est) missingFields.push("Establishment ID"); - if (!factory) missingFields.push("Factory Name"); - if (!email) missingFields.push("Email"); - if (!emirate) missingFields.push("Emirate"); - if (!employment) missingFields.push("Total Employment"); - - if (missingFields.length > 0) { - errors.push({ - row: rowNum, - error: `Missing Data: ${missingFields.join(", ")} ${missingFields.length === 1 ? "is" : "are"} required.` - }); + if (missing.length) { + errors.push({ row: rowNum, error: `Missing required fields: ${missing.join(", ")}` }); continue; } - if (isNaN(employment) || !/^\d+$/.test(employment)) { - errors.push({ - row: rowNum, - error: "Total Employment must be numeric." - }); + if (!/^[^\s@]+@[^\s@]+\.[^\s@]+$/.test(email)) { + errors.push({ row: rowNum, error: "Invalid Email format" }); continue; } - const emailRegex = /^[^\s@]+@[^\s@]+\.[^\s@]+$/; - - if (!emailRegex.test(email)) { - errors.push({ - row: rowNum, - error: "Invalid Email: Please check the Email address." - }); - continue; - } // FILE-level duplicates const estKey = est.trim(); const factoryKey = factory.trim().toLowerCase(); @@ -1286,7 +1246,7 @@ exports.establishmentBulkUpload = async (req, res) => { fileEmailSet.add(emailKey); // DB-level duplicates - if (existingEstIdSet.has(estKey)) { + if (existingEstSet.has(estKey)) { errors.push({ row: rowNum, error: `Establishment ID already exists: ${est}` }); continue; } @@ -1306,116 +1266,127 @@ exports.establishmentBulkUpload = async (req, res) => { continue; } - // ------------------------------------------------------- - // HS CODE VALIDATION - // ------------------------------------------------------- + // HS Code validation const hsCols = Object.keys(r).filter(k => k.startsWith("hs")); const hsRaw = hsCols.map(k => r[k]).filter(Boolean); if (hsRaw.length === 0) { - errors.push({ row: rowNum, error: "At least one HS Code is required" }); + errors.push({ row: rowNum, error: "At least one HS code is required" }); continue; } - // Clean & remove leading zeros - const cleaned = hsRaw.map(v => - v.replace(/[^0-9]/g, "").replace(/^0+/, "") - ); + const cleaned = hsRaw.map(h => normalizeHS(h)); + const invalid = cleaned.filter(c => !productMap[c]); - // Check duplicates in same row - const rowDuplicates = cleaned.filter((c, idx) => cleaned.indexOf(c) !== idx); - if (rowDuplicates.length > 0) { - errors.push({ - row: rowNum, - error: `Duplicate HS Code(s) in the same row: ${[...new Set(rowDuplicates)].join(", ")}` - }); + if (invalid.length > 0) { + errors.push({ row: rowNum, error: `Invalid HS Code(s): ${invalid.join(", ")}` }); continue; } - // DB-level HS Code validation - const invalid = []; - - for (const raw of hsRaw) { - const cleaned = normalizeHS(raw); - - if (cleaned.length < 1 || cleaned.length > 10) { - invalid.push(raw); - continue; - } - - if (!productMap[cleaned]) { - invalid.push(raw); - } - } - - if (invalid.length > 0) { - errors.push({ - row: rowNum, - error: `HS Code not found in database: ${invalid.join(", ")}` - }); - continue; - } - prepared.push({ + r, + rowNum, est, factory, email, emirateId, - employment: parseInt(employment), - productIds: cleaned.map(c => productMap[c]), - rowNum + productIds: cleaned.map(c => productMap[c]) }); } // ------------------------------------------------------- - // STOP IF ERRORS + // STOP IF ANY ERRORS FROM PHASE 1 // ------------------------------------------------------- if (errors.length > 0) { - logger.error("Establishment Upload Failed: " + JSON.stringify(errors)); fs.unlinkSync(filePath); return res.status(400).send({ status: "failed", - message: "Invalid data. Please check the instructions given and upload again.", + message: "Invalid data. Please check and upload again.", errors }); } // ------------------------------------------------------- - // INSERT VALID RECORDS + // PHASE 2: CITY/TOWN VALIDATION (OPTIONAL BUT MUST BE VALID) // ------------------------------------------------------- - for (const rec of prepared) { + for (const p of prepared) { + const r = p.r; + const name = r.city_town?.trim().toLowerCase() || ""; + + if (name) { + const id = cityTownMap[name]; + if (!id) { + fs.unlinkSync(filePath); + return res.status(400).send({ + status: "failed", + message: "Invalid City/Town found. Upload stopped.", + errors: [{ row: p.rowNum, error: `Invalid City/Town: ${r.city_town}` }] + }); + } + r.city_town_id = id; + } else { + r.city_town_id = null; + } + } + + // ------------------------------------------------------- + // PHASE 3: INSERT INTO DATABASE + // ------------------------------------------------------- + for (const p of prepared) { + const r = p.r; + const est = await Establishment.create({ - establishment_code: rec.est, - factory_name: rec.factory, - establishment_contact_email: rec.email, - establishment_emirate_id: rec.emirateId, - total_employees: rec.employment, + establishment_code: p.est, + factory_name: p.factory, + establishment_contact_email: p.email, + establishment_emirate_id: p.emirateId, + + permanent_factory_code: r.permanent_factory_code, + industry_code: r.industry_code_business_register, + industry_code_production: r.industry_code_current_production, + industry_code_mismatch_remarks: r.industry_code_mismatch_remarks, + description: r.description, + establishment_address: r.establishment_address, + establishment_city_town_id: r.city_town_id, + establishment_postal_code: r.postal_code, + establishment_po_box: r.po_box, + establishment_makani_number: r.makani_number, + establishment_contact_person_name: r.contact_person_name, + establishment_contact_person_designation: r.contact_person_designation, + establishment_mobile_number: r.mobile_number, + establishment_website: r.website, + + emirati_male: Number(r.number_of_emirati_male || 0), + emirati_female: Number(r.number_of_emirati_female || 0), + non_emirati_male: Number(r.number_of_non_emirati_male || 0), + non_emirati_female: Number(r.number_of_non_emirati_female || 0), + + total_emirati: + Number(r.number_of_emirati_male || 0) + + Number(r.number_of_emirati_female || 0), + + total_employees: + Number(r.number_of_emirati_male || 0) + + Number(r.number_of_emirati_female || 0) + + Number(r.number_of_non_emirati_male || 0) + + Number(r.number_of_non_emirati_female || 0), + created_by: req.user.id }); - const autoPassword = Math.random().toString(36).slice(-10); - const hashedPassword = await bcrypt.hash(autoPassword, 10); - const user = await EstablishmentUser.create({ + const autoPassword = Math.random().toString(36).slice(-10); + const hashed = await bcrypt.hash(autoPassword, 10); + + await EstablishmentUser.create({ establishment_id: est.id, - name: rec.factory, - email: rec.email, - password: hashedPassword, + name: r.user_name, + email: p.email, + password: hashed, created_by: req.user.id }); - const placeHolderData = { - contact_name: rec.factory || rec.est, - portal_url: process.env.FE_BASE_URL, - username: rec.email, - password: autoPassword, - support_email: process.env.SUPPORT_EMAIL, - support_phone: process.env.SUPPORT_PHONE - }; - - await sendEmailService(rec.email, "establishment_user_creation_to_user", placeHolderData); - await EstablishmentProduct.bulkCreate( - rec.productIds.map(pid => ({ + p.productIds.map(pid => ({ establishment_id: est.id, product_id: pid, created_by: req.user.id @@ -1436,15 +1407,375 @@ exports.establishmentBulkUpload = async (req, res) => { }); } catch (err) { - logger.error("Establishment Import Fatal Error: " + err.message); - + logger.error("Fatal Error: " + err.message); return res.status(500).send({ status: "failed", - message: "Invalid data. Please check the instructions given and upload again." + message: "Unexpected error occurred." }); } }; + +// exports.establishmentBulkUpload = async (req, res) => { +// try { +// if (!req.file) { +// return res.status(400).send({ +// status: "failed", +// message: "No file uploaded." +// }); +// } + +// const filePath = req.file.path; + +// if (!req.file.originalname.toLowerCase().endsWith(".csv")) { +// fs.unlinkSync(filePath); +// return res.status(400).send({ +// status: "failed", +// message: "Invalid file type. Only CSV allowed." +// }); +// } + +// const rows = []; + +// // ------------------------------------------------------- +// // READ CSV AND SKIP EMPTY ROWS +// // ------------------------------------------------------- +// await new Promise((resolve, reject) => { +// fs.createReadStream(filePath) +// .pipe(csv()) +// .on("data", (rawRow) => { +// const row = {}; + +// // Normalize headers +// for (const key in rawRow) { +// const normalizedKey = key.replace(/[\s\W]+/g, "_") +// .trim() +// .toLowerCase(); + +// row[normalizedKey] = rawRow[key]?.trim() || ""; +// } + +// const cleanedValues = Object.values(row).map(v =>(v || "").replace(/\s+/g, "").trim()); +// const isEmpty = cleanedValues.every(v => v === ""); +// if (isEmpty) return; + +// rows.push(row); +// }) +// .on("end", resolve) +// .on("error", reject); +// }); + +// if (rows.length === 0) { +// fs.unlinkSync(filePath); +// return res.status(400).send({ +// status: "failed", +// message: "CSV file is empty.", +// errors: [{ error: "Uploaded file is empty." }] +// }); +// } +// // ------------------------------------------------------- +// // REQUIRED HEADER CHECK +// // ------------------------------------------------------- +// const required = [ +// "establishment_id", +// "factory_name", +// "email", +// "emirate", +// "total_employment" +// ]; + +// const firstRowKeys = Object.keys(rows[0]); + +// const missing = required.filter(r => !firstRowKeys.includes(r)); +// if (missing.length > 0) { +// fs.unlinkSync(filePath); +// return res.status(400).send({ +// status: "failed", +// message: "Invalid data. Please check the instructions given and upload again.", +// errors: [ +// { error: "Missing required columns: " + missing.join(", ") } +// ] +// }); +// } + +// // ------------------------------------------------------- +// // LOAD REFERENCE TABLES +// // ------------------------------------------------------- +// const emirates = await Emirate.findAll({ attributes: ["id", "name"] }); +// const emirateMap = {}; +// emirates.forEach(e => { +// emirateMap[e.name.trim().toLowerCase()] = e.id; +// }); + +// function normalizeHS(val) { +// if (!val) return ""; + +// return val +// .toString() +// .normalize("NFKD") +// .replace(/[^\d]/g, "") +// .trim(); +// } + +// const products = await Product.findAll({ attributes: ["id", "hs_code"] }); + +// const productMap = {}; + +// products.forEach(p => { +// const cleaned = normalizeHS(p.hs_code); + +// // Only store valid HS codes 1–10 digits +// if (cleaned.length >= 1 && cleaned.length <= 10) { +// productMap[cleaned] = p.id; +// } +// }); +// const existingEsts = await Establishment.findAll({ +// attributes: ["establishment_code", "factory_name", "establishment_contact_email"] +// }); + +// const existingEstIdSet = new Set(existingEsts.map(e => e.establishment_code)); +// const existingFactorySet = new Set(existingEsts.map(e => e.factory_name.toLowerCase())); +// const existingEmailSet = new Set(existingEsts.map(e => e.establishment_contact_email.toLowerCase())); + +// // ------------------------------------------------------- +// // FILE-LEVEL DUPLICATE TRACKING +// // ------------------------------------------------------- +// const errors = []; +// const fileEstSet = new Set(); +// const fileFactorySet = new Set(); +// const fileEmailSet = new Set(); + +// const prepared = []; + +// // ------------------------------------------------------- +// // VALIDATE EACH ROW +// // ------------------------------------------------------- +// for (let i = 0; i < rows.length; i++) { +// const r = rows[i]; +// const rowNum = i + 2; + +// const est = r.establishment_id; +// const factory = r.factory_name; +// const email = r.email; +// const emirate = r.emirate; +// const employment = r.total_employment; + +// // Required validation +// const missingFields = []; + +// if (!est) missingFields.push("Establishment ID"); +// if (!factory) missingFields.push("Factory Name"); +// if (!email) missingFields.push("Email"); +// if (!emirate) missingFields.push("Emirate"); +// if (!employment) missingFields.push("Total Employment"); + +// if (missingFields.length > 0) { +// errors.push({ +// row: rowNum, +// error: `Missing Data: ${missingFields.join(", ")} ${missingFields.length === 1 ? "is" : "are"} required.` +// }); +// continue; +// } + +// if (isNaN(employment) || !/^\d+$/.test(employment)) { +// errors.push({ +// row: rowNum, +// error: "Total Employment must be numeric." +// }); +// continue; +// } + +// const emailRegex = /^[^\s@]+@[^\s@]+\.[^\s@]+$/; + +// if (!emailRegex.test(email)) { +// errors.push({ +// row: rowNum, +// error: "Invalid Email: Please check the Email address." +// }); +// continue; +// } +// // FILE-level duplicates +// const estKey = est.trim(); +// const factoryKey = factory.trim().toLowerCase(); +// const emailKey = email.trim().toLowerCase(); + +// if (fileEstSet.has(estKey)) { +// errors.push({ row: rowNum, error: `Duplicate Establishment ID in file: ${est}` }); +// continue; +// } +// if (fileFactorySet.has(factoryKey)) { +// errors.push({ row: rowNum, error: `Duplicate Factory Name in file: ${factory}` }); +// continue; +// } +// if (fileEmailSet.has(emailKey)) { +// errors.push({ row: rowNum, error: `Duplicate Email in file: ${email}` }); +// continue; +// } + +// fileEstSet.add(estKey); +// fileFactorySet.add(factoryKey); +// fileEmailSet.add(emailKey); + +// // DB-level duplicates +// if (existingEstIdSet.has(estKey)) { +// errors.push({ row: rowNum, error: `Establishment ID already exists: ${est}` }); +// continue; +// } +// if (existingFactorySet.has(factoryKey)) { +// errors.push({ row: rowNum, error: `Factory Name already exists: ${factory}` }); +// continue; +// } +// if (existingEmailSet.has(emailKey)) { +// errors.push({ row: rowNum, error: `Email already exists: ${email}` }); +// continue; +// } + +// // Emirate validation +// const emirateId = emirateMap[emirate.toLowerCase()]; +// if (!emirateId) { +// errors.push({ row: rowNum, error: `Invalid Emirate: ${emirate}` }); +// continue; +// } + +// // ------------------------------------------------------- +// // HS CODE VALIDATION +// // ------------------------------------------------------- +// const hsCols = Object.keys(r).filter(k => k.startsWith("hs")); +// const hsRaw = hsCols.map(k => r[k]).filter(Boolean); + +// if (hsRaw.length === 0) { +// errors.push({ row: rowNum, error: "At least one HS Code is required" }); +// continue; +// } + +// // Clean & remove leading zeros +// const cleaned = hsRaw.map(v => +// v.replace(/[^0-9]/g, "").replace(/^0+/, "") +// ); + +// // Check duplicates in same row +// const rowDuplicates = cleaned.filter((c, idx) => cleaned.indexOf(c) !== idx); +// if (rowDuplicates.length > 0) { +// errors.push({ +// row: rowNum, +// error: `Duplicate HS Code(s) in the same row: ${[...new Set(rowDuplicates)].join(", ")}` +// }); +// continue; +// } + +// // DB-level HS Code validation +// const invalid = []; + +// for (const raw of hsRaw) { +// const cleaned = normalizeHS(raw); + +// if (cleaned.length < 1 || cleaned.length > 10) { +// invalid.push(raw); +// continue; +// } + +// if (!productMap[cleaned]) { +// invalid.push(raw); +// } +// } + +// if (invalid.length > 0) { +// errors.push({ +// row: rowNum, +// error: `HS Code not found in database: ${invalid.join(", ")}` +// }); +// continue; +// } + +// prepared.push({ +// est, +// factory, +// email, +// emirateId, +// employment: parseInt(employment), +// productIds: cleaned.map(c => productMap[c]), +// rowNum +// }); +// } + +// // ------------------------------------------------------- +// // STOP IF ERRORS +// // ------------------------------------------------------- +// if (errors.length > 0) { +// logger.error("Establishment Upload Failed: " + JSON.stringify(errors)); +// fs.unlinkSync(filePath); +// return res.status(400).send({ +// status: "failed", +// message: "Invalid data. Please check the instructions given and upload again.", +// errors +// }); +// } + +// // ------------------------------------------------------- +// // INSERT VALID RECORDS +// // ------------------------------------------------------- +// for (const rec of prepared) { +// const est = await Establishment.create({ +// establishment_code: rec.est, +// factory_name: rec.factory, +// establishment_contact_email: rec.email, +// establishment_emirate_id: rec.emirateId, +// total_employees: rec.employment, +// created_by: req.user.id +// }); + +// const autoPassword = Math.random().toString(36).slice(-10); +// const hashedPassword = await bcrypt.hash(autoPassword, 10); +// const user = await EstablishmentUser.create({ +// establishment_id: est.id, +// name: rec.factory, +// email: rec.email, +// password: hashedPassword, +// created_by: req.user.id +// }); + +// const placeHolderData = { +// contact_name: rec.factory || rec.est, +// portal_url: process.env.FE_BASE_URL, +// username: rec.email, +// password: autoPassword, +// support_email: process.env.SUPPORT_EMAIL, +// support_phone: process.env.SUPPORT_PHONE +// }; + +// await sendEmailService(rec.email, "establishment_user_creation_to_user", placeHolderData); + +// await EstablishmentProduct.bulkCreate( +// rec.productIds.map(pid => ({ +// establishment_id: est.id, +// product_id: pid, +// created_by: req.user.id +// })) +// ); +// } + +// fs.unlinkSync(filePath); + +// return res.status(200).send({ +// status: "success", +// message: `${prepared.length} establishments inserted successfully.`, +// summary: { +// total_records: rows.length, +// imported: prepared.length, +// errors: [] +// } +// }); + +// } catch (err) { +// logger.error("Establishment Import Fatal Error: " + err.message); + +// return res.status(500).send({ +// status: "failed", +// message: "Invalid data. Please check the instructions given and upload again." +// }); +// } +// }; + // exports.establishmentBulkUpload = async (req, res) => { // try { // if (!req.file) {