diff --git a/app/controllers/establishment.controller.js b/app/controllers/establishment.controller.js index 28cd8e0..6ce4fbc 100644 --- a/app/controllers/establishment.controller.js +++ b/app/controllers/establishment.controller.js @@ -889,7 +889,6 @@ exports.establishmentBulkUpload = async (req, res) => { row[normalizedKey] = rawRow[key]?.trim() || null; continue; } - // Fallback - keep other columns too row[normalizedKey] = rawRow[key]?.trim() || null; } @@ -1079,34 +1078,28 @@ exports.establishmentBulkUpload = async (req, res) => { .map((hc) => hc.replace(/[^0-9]/g, "").replace(/^0+/, "")) .filter(Boolean); // remove empty after cleaning - // Map to product IDs - const mappedProducts = cleanedHs - .map((c) => productMap[c]) - .filter(Boolean); + const mappedProducts = cleanedHs.map((c) => productMap[c] || null); - // Validate mapping - if (mappedProducts.length === 0) { + // Find which HS codes do NOT match product master + const notMapped = cleanedHs.filter((c, i) => mappedProducts[i] === null); + + if (notMapped.length > 0) { + errors.push({ + row: rowNumber, + error: "Some HS Codes do not exist in product master.", + not_mapped: notMapped + }); + continue; + } + + if (mappedProducts.every((id) => id === null)) { errors.push({ row: rowNumber, error: "None of the provided HS Codes match existing products.", hs_codes: cleanedHs }); continue; - } - - if (mappedProducts.length === 0) { - errors.push({ row: rowNum, error: "No HS codes matched product list" }); - continue; - } - // Validation: at least one HS must match existing product - if (mappedProducts.length === 0) { - errors.push({ - row: rowNumber, - error: "None of the provided HS Codes match existing products.", - hs_codes: cleanedHs, - }); - continue; - } + } // parse employment const employment = parseInt(employmentRaw) || 0; @@ -1205,5 +1198,393 @@ exports.establishmentBulkUpload = async (req, res) => { } }; +// exports.establishmentBulkUpload = async (req, res) => { +// try { +// // Basic file & user validations +// if (!req.file) { +// return res.status(400).send({ status: "failed", message: "No file uploaded" }); +// } +// const filePath = req.file.path; + +// if (!req.file.originalname.toLowerCase().endsWith(".csv")) { +// if (fs.existsSync(filePath)) fs.unlinkSync(filePath); +// return res.status(400).send({ status: "failed", message: "Invalid file type. Only CSV allowed." }); +// } + +// if (!req.user?.id || isNaN(req.user.id)) { +// if (fs.existsSync(filePath)) fs.unlinkSync(filePath); +// return res.status(400).send({ status: "failed", message: "Invalid or missing User Id." }); +// } + +// const mode = (req.body.mode || "add").toLowerCase(); +// if (mode !== "add") { +// if (fs.existsSync(filePath)) fs.unlinkSync(filePath); +// return res.status(400).send({ status: "failed", message: "Only 'Add Only' mode is supported currently." }); +// } + +// const stats = fs.statSync(filePath); +// if (stats.size === 0) { +// if (fs.existsSync(filePath)) fs.unlinkSync(filePath); +// return res.status(400).send({ status: "failed", message: "Uploaded file is empty." }); +// } + +// // Read CSV into memory (array of normalized rows) +// const rows = []; +// fs.createReadStream(filePath) +// .pipe(csv()) +// .on("data", (rawRow) => { +// // Normalize headers and map to canonical keys +// const row = {}; +// for (const key in rawRow) { +// const normalizedKey = key.replace(/[\s\W]+/g, "_").trim().toLowerCase(); + +// // Standard canonical keys +// if (normalizedKey === "establishment_id" || normalizedKey === "establishment_code" || normalizedKey.startsWith("establishment")) { +// row["Establishment Id"] = rawRow[key]?.trim() || null; +// continue; +// } + +// if (normalizedKey.includes("factory name") || normalizedKey.includes("factory_name") || normalizedKey.includes("Factory Name")) { +// row["Factory Name"] = rawRow[key]?.trim() || null; +// continue; +// } + +// if (normalizedKey.includes("Email") || normalizedKey.includes("email") ) { +// row["Email"] = rawRow[key]?.trim() || null; +// continue; +// } + +// if (normalizedKey.includes("Emirate") || normalizedKey.includes("emirate") ) { +// row["Emirate"] = rawRow[key]?.trim() || null; +// continue; +// } + +// if (normalizedKey.includes("total employment") || normalizedKey.includes("total_employment") || normalizedKey.includes("total employments")) { +// row["Total Employment"] = rawRow[key]?.trim() || null; +// continue; +// } + +// // HS code columns: keep any header that contains 'hs' (hs_code, hs_code_1, hs1, etc.) +// if (/^hs|hs_code|hs_code_\d|hs\d/i.test(normalizedKey) || normalizedKey.includes("hs")) { +// // keep the exact normalized header so we can iterate later +// row[normalizedKey] = rawRow[key]?.trim() || null; +// continue; +// } + +// // Fallback - keep other columns too +// row[normalizedKey] = rawRow[key]?.trim() || null; +// } + +// rows.push(row); +// }) +// .on("end", async () => { +// try { +// if (!rows.length) { +// if (fs.existsSync(filePath)) fs.unlinkSync(filePath); +// return res.status(400).send({ status: "failed", message: "CSV file is empty or invalid." }); +// } + +// // Check required canonical columns exist in the header (based on first row keys) +// const firstRowKeys = Object.keys(rows[0]); +// const required = ["Establishment Id", "Factory Name", "Email", "Emirate", "Total Employment"]; +// const missing = required.filter((k) => !firstRowKeys.includes(k)); +// if (missing.length > 0) { +// if (fs.existsSync(filePath)) fs.unlinkSync(filePath); +// return res.status(400).send({ +// status: "failed", +// message: `Missing required columns: ${missing.join(", ")}`, +// }); +// } + +// // Prepare maps: emirates and products +// const emirates = await Emirate.findAll({ attributes: ["id", "name"] }); +// const emirateMap = {}; +// emirates.forEach((e) => { +// if (e.name) emirateMap[e.name.trim().toLowerCase()] = e.id; +// }); + +// // Build product map by normalized hs_code (digits only, no leading zeros) +// const products = await Product.findAll({ attributes: ["id", "hs_code"] }); +// const productMap = {}; +// products.forEach((p) => { +// if (!p.hs_code) return; +// const normalized = String(p.hs_code).replace(/[^0-9]/g, "").replace(/^0+/, ""); +// if (normalized) productMap[normalized] = p.id; +// }); + + +// // Existing establishment codes in DB +// const existingEsts = await Establishment.findAll({ +// attributes: ["establishment_code", "factory_name", "establishment_contact_email"] +// }); + +// const existingEstIdSet = new Set(existingEsts.map((e) => e.establishment_code)); +// const existingFactorySet = new Set(existingEsts.map((e) => e.factory_name?.trim().toLowerCase())); +// const existingEmailSet = new Set(existingEsts.map((e) => e.establishment_contact_email?.trim().toLowerCase())); + +// const errors = []; +// const fileDuplicates = new Set(); +// const seenEstIds = new Set(); +// const toInsert = []; + +// // Process each row: validate & prepare +// for (let i = 0; i < rows.length; i++) { +// const raw = rows[i]; +// const rowNumber = i + 1; + +// const estId = raw["Establishment Id"]?.trim(); +// const factory = raw["Factory Name"]?.trim(); +// const email = raw["Email"]?.trim(); +// const emirateRaw = raw["Emirate"]?.trim(); +// const employmentRaw = raw["Total Employment"]?.trim(); + +// // Required fields +// if (!estId || !factory || !email || !emirateRaw) { +// errors.push({ +// row: rowNumber, +// error: "Missing required fields (Establishment Id, Factory Name, Email, Emirate).", +// }); +// continue; +// } + +// //Checking repeating data +// const estIdRowMap = {}; +// const factoryRowMap = {}; +// const emailRowMap = {}; + +// // During row loop — replace your tracking code with this: +// const keyEst = estId?.trim(); +// const keyFactory = factory?.trim().toLowerCase(); +// const keyEmail = email?.trim().toLowerCase(); + +// // Helper for inserting row numbers +// const pushRow = (map, key) => +// key ? (map[key] = map[key] ? [...map[key], rowNumber] : [rowNumber]) : null; + +// // Track all 3 +// pushRow(estIdRowMap, keyEst); +// pushRow(factoryRowMap, keyFactory); +// pushRow(emailRowMap, keyEmail); + +// const duplicateDetails = []; + +// [ +// ["Establishment Id", estIdRowMap], +// ["Factory Name", factoryRowMap], +// ["Email", emailRowMap], +// ].forEach(([field, map]) => { +// Object.entries(map).forEach(([value, rows]) => { +// if (rows.length > 1) { +// duplicateDetails.push({ +// field, +// value, +// rows +// }); +// } +// }); +// }); +// if (duplicateDetails.length > 0) { +// if (fs.existsSync(filePath)) fs.unlinkSync(filePath); +// return res.status(400).send({ +// status: "failed", +// message: "Repeated values found in uploaded file.", +// duplicates: duplicateDetails, +// }); +// } + +// const hsRawCodes = []; +// for (const k of Object.keys(raw)) { +// if (/^hs|hs_code|hs_code_\d|hs\d/i.test(k) || k.includes("hs")) { +// const v = raw[k]; +// if (v && String(v).trim()) { +// hsRawCodes.push(String(v).trim()); +// } +// } +// } + +// // Validation: at least one HS code must be present +// if (hsRawCodes.length === 0) { +// errors.push({ +// row: rowNumber, +// error: "Missing HS Code columns or no HS Code values provided.", +// }); +// continue; +// } + +// const seenEstIds = new Set(); +// const seenFactories = new Set(); +// const seenEmails = new Set(); + +// if (seenEstIds.has(keyEst)) { +// fileDuplicates.add(`Establishment ID: ${keyEst}`); +// continue; +// } +// if (seenFactories.has(keyFactory)) { +// fileDuplicates.add(`Factory Name: ${factory}`); +// continue; +// } +// if (seenEmails.has(keyEmail)) { +// fileDuplicates.add(`Email: ${email}`); +// continue; +// } + +// seenEstIds.add(keyEst); +// seenFactories.add(keyFactory); +// seenEmails.add(keyEmail); + +// if (existingEstIdSet.has(keyEst)) { +// fileDuplicates.add(`Establishment ID already exists: ${keyEst}`); +// continue; +// } +// if (existingFactorySet.has(keyFactory)) { +// fileDuplicates.add(`Factory Name already exists: ${factory}`); +// continue; +// } +// if (existingEmailSet.has(keyEmail)) { +// fileDuplicates.add(`Email already exists: ${email}`); +// continue; +// } + +// // Emirate mapping +// const emirateId = emirateMap[emirateRaw.toLowerCase()]; +// if (!emirateId) { +// errors.push({ +// row: rowNumber, +// error: `Invalid Emirate: '${emirateRaw}'`, +// }); +// continue; +// } + +// // Clean HS codes: remove non-digits and leading zeros +// const cleanedHs = hsRawCodes +// .map((hc) => hc.replace(/[^0-9]/g, "").replace(/^0+/, "")) +// .filter(Boolean); // remove empty after cleaning + +// // Map to product IDs +// const mappedProducts = cleanedHs +// .map((c) => productMap[c]) +// .filter(Boolean); + +// // Validate mapping +// if (mappedProducts.length === 0) { +// errors.push({ +// row: rowNumber, +// error: "None of the provided HS Codes match existing products.", +// hs_codes: cleanedHs +// }); +// continue; +// } + +// if (mappedProducts.length === 0) { +// errors.push({ row: rowNum, error: "No HS codes matched product list" }); +// continue; +// } +// // Validation: at least one HS must match existing product +// if (mappedProducts.length === 0) { +// errors.push({ +// row: rowNumber, +// error: "None of the provided HS Codes match existing products.", +// hs_codes: cleanedHs, +// }); +// continue; +// } + +// // parse employment +// const employment = parseInt(employmentRaw) || 0; + +// toInsert.push({ +// estCode: estId, +// factory, +// email, +// emirateId, +// employment, +// productIds: [...new Set(mappedProducts)], +// rowIndex: rowNumber, +// }); +// } + +// // If file duplicates found (either in file or in DB), return error - consistent with product controller +// if (fileDuplicates.size > 0) { +// if (fs.existsSync(filePath)) fs.unlinkSync(filePath); +// return res.status(400).send({ +// status: "failed", +// message: "Duplicate Establishment IDs found (in file or already exist in system). Resolve and re-upload.", +// duplicates: [...fileDuplicates], +// }); +// } + +// // Insert prepared establishments and associated products +// let inserted = 0; +// for (const rec of toInsert) { +// try { +// const createdEst = await Establishment.create({ +// establishment_code: rec.estCode, +// factory_name: rec.factory, +// establishment_contact_email: rec.email, +// establishment_emirate_id: rec.emirateId, +// total_employees: rec.employment, +// created_by: req.user.id, +// created_at: new Date(), +// }); + +// // prepare bulk create payload for establishment_products +// const payload = rec.productIds.map((pid) => ({ +// establishment_id: createdEst.id, +// product_id: pid, +// created_by: req.user.id, +// created_at: new Date(), +// is_active: true, +// })); + +// await EstablishmentProduct.bulkCreate(payload); +// inserted++; +// } catch (errInner) { +// // Collect row-level DB insert errors for partial success reporting +// errors.push({ +// row: rec.rowIndex, +// error: `DB insert error: ${errInner.message}`, +// }); +// } +// } + +// // cleanup uploaded file +// if (fs.existsSync(filePath)) fs.unlinkSync(filePath); + +// // Build final response to match product import style +// let finalStatus = "success"; +// let message = `${inserted} establishments inserted successfully.`; + +// if (errors.length > 0) { +// finalStatus = inserted > 0 ? "partial_success" : "failed"; +// if (finalStatus === "partial_success") { +// message = `${inserted} establishments inserted, ${errors.length} rows failed.`; +// } else { +// message = `No establishments imported. ${errors.length} validation errors found.`; +// } +// } + +// return res.status(200).send({ +// status: finalStatus, +// message, +// summary: { +// total_records: rows.length, +// imported: inserted, +// errors, +// }, +// }); +// } catch (err) { +// if (fs.existsSync(filePath)) fs.unlinkSync(filePath); +// return res.status(500).send({ status: "failed", message: err.message }); +// } +// }) +// .on("error", (err) => { +// if (fs.existsSync(filePath)) fs.unlinkSync(filePath); +// return res.status(500).send({ status: "failed", message: err.message }); +// }); +// } catch (error) { +// return res.status(500).send({ status: "failed", message: error.message }); +// } +// }; +