FEAT_TPA_API_DATA_STORE_JOB
This commit is contained in:
velz
parent
0eea675a9f
commit
5558e95554
@ -766,6 +766,7 @@ $routes->get('IRSubmission','MediAssistApiController::IRSubmission');
|
||||
// API service
|
||||
$routes->post("ecardRequest", "ApiServiceController::ecardRequest");
|
||||
$routes->post("getTPAID", "ApiServiceController::getTPAID");
|
||||
// $routes->get("getTPAID", "ApiServiceController::getTPAID");
|
||||
$routes->get('fileDownload','MediAssistApiController::fileDownload');
|
||||
|
||||
|
||||
|
||||
@ -14,6 +14,8 @@ use App\Controllers\ICICILombardController;
|
||||
use App\Controllers\MediAssistApiController;
|
||||
use App\Models\BatchFileModel;
|
||||
use App\Models\FileModel;
|
||||
use App\Helpers\TPADataCompareHelper;
|
||||
use App\Helpers\TPADataCompareHelper2;
|
||||
|
||||
class ApiServiceController extends BaseController
|
||||
{
|
||||
@ -154,6 +156,19 @@ class ApiServiceController extends BaseController
|
||||
public function getTPAID()
|
||||
{
|
||||
|
||||
// helper('TPADataCompareHelper');
|
||||
// print_rr(json_decode(file_get_contents(WRITEPATH.'/tmp/medi.json'),true));die();
|
||||
// $report = TPADataCompareHelper::compareDbVsMediassist(
|
||||
// $report = TPADataCompareHelper2::compareDbVsMediassistGrouped(
|
||||
// json_decode(file_get_contents(WRITEPATH.'/tmp/db.json'),true),
|
||||
// json_decode(file_get_contents(WRITEPATH.'/tmp/medi.json'),true)
|
||||
// );
|
||||
// print_rr($report);
|
||||
// die();
|
||||
|
||||
|
||||
|
||||
//START OF THE PROGRAM
|
||||
log_message('error', "getTPAID payloads :" . json_encode($this->request->getPost() ?? []));
|
||||
$tpa_id = $this->request->getPost('tpa_id') ?? null;
|
||||
$policy_no = $this->request->getPost('policy_no') ?? null;
|
||||
|
||||
@ -190,6 +190,9 @@ class JobWorker extends AdminController
|
||||
'initiateWellnessOnboardJob' => [
|
||||
'type' => 'CC', // Handler Category
|
||||
'handler' => 'App\Controllers\EmployeeController',
|
||||
], 'saveMediAssitAPIData' => [
|
||||
'type' => 'CC', // Handler Category
|
||||
'handler' => 'App\Controllers\MediAssistApiController',
|
||||
],
|
||||
];
|
||||
|
||||
|
||||
@ -5,10 +5,12 @@ namespace App\Controllers;
|
||||
use App\Controllers\BaseController;
|
||||
use App\Models\BatchFileModel;
|
||||
use App\Models\EmployeePolicyModel;
|
||||
use App\Models\TpaApiDataModel;
|
||||
use CodeIgniter\HTTP\ResponseInterface;
|
||||
use CodeIgniter\HTTP\IncomingRequest;
|
||||
use CodeIgniter\HTTP\RequestInterface;
|
||||
use CodeIgniter\API\ResponseTrait;
|
||||
use App\Controllers\Jobs;
|
||||
|
||||
class MediAssistApiController extends BaseController
|
||||
{
|
||||
@ -203,8 +205,18 @@ class MediAssistApiController extends BaseController
|
||||
|
||||
public function GetBenefDetails($requestData)
|
||||
{
|
||||
helper('api');
|
||||
// helper('api');
|
||||
// helper('TPADataCompareHelper');
|
||||
// $report = EmployeeCompareHelper::compareDbVsMediassist(
|
||||
// $dbRows,
|
||||
// $mediassistAPIdata
|
||||
// );
|
||||
// print_rr($report);
|
||||
// die();
|
||||
// print_r($requestData);die();
|
||||
|
||||
|
||||
//START OF THE PROGRAM
|
||||
$function_calling_type = $requestData['return_type'] ?? 'job';
|
||||
|
||||
try {
|
||||
@ -266,7 +278,6 @@ class MediAssistApiController extends BaseController
|
||||
return $this->respond(['status' => false, 'message' => 'employeePolicyData not found']);
|
||||
}
|
||||
}
|
||||
|
||||
$allBenef = [];
|
||||
$startIndex = 0;
|
||||
$range = 100;
|
||||
@ -326,6 +337,19 @@ class MediAssistApiController extends BaseController
|
||||
$startIndex += $range;
|
||||
} while ($startIndex < $totalCount);
|
||||
|
||||
//save API data as JSON for analysis
|
||||
$json = json_encode($allBenef, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES);
|
||||
// $filename = time() . '.json';
|
||||
$filePath = WRITEPATH . 'tmp/'.time().'_'.$requestData['file_id'].'.json';
|
||||
file_put_contents($filePath, $json);
|
||||
|
||||
//call a job for dump JSON data to DB
|
||||
$job_details = new Jobs();
|
||||
$r = Jobs::addJob(['job_name' => 'saveMediAssitAPIData', 'payload' => [
|
||||
'file_id' => $requestData['file_id'],
|
||||
'json_file_path' => $filePath ]]);
|
||||
|
||||
|
||||
// now update DB
|
||||
$updated = 0;
|
||||
$employee_policy_ids = [];
|
||||
@ -334,11 +358,10 @@ class MediAssistApiController extends BaseController
|
||||
$hasMatchForThisPolicy = false;
|
||||
|
||||
foreach ($allBenef as $row) {
|
||||
|
||||
if (
|
||||
strtolower(trim($policy_data['name'] ?? '')) == strtolower(trim($row['benefName'] ?? '')) &&
|
||||
($policy_data['emp_code'] ?? '') == ($row['priBenefEmpCode'] ?? '') &&
|
||||
strtolower(trim($policy_data['relationship'] ?? '')) == strtolower(trim($row['relName'] ?? '')) &&
|
||||
strtolower(trim($policy_data['relationship'] ?? '')) == strtolower(trim(str_replace('-', ' ', $row['relName'] ?? ''))) &&
|
||||
($policy_data['gender'] ?? '') == ($row['benefSex'] ?? '') &&
|
||||
($policy_data['dob'] ?? '') == (change_date_format($row['benefDOB'], 'd/m/Y H:i:s') ?? '')
|
||||
) {
|
||||
@ -721,6 +744,54 @@ class MediAssistApiController extends BaseController
|
||||
];
|
||||
}
|
||||
|
||||
public function saveMediAssitAPIData($array)
|
||||
{
|
||||
$file_id = $array['file_id'];
|
||||
$json = file_get_contents($array['json_file_path']);
|
||||
$records = json_decode($json, true);
|
||||
// log_message('error','saveMediAssitAPIData' . json_encode($array));//die();
|
||||
$file_model = new BatchFileModel();
|
||||
$file_info = $file_model->where('id', $file_id)->find();
|
||||
// dd($file_info);
|
||||
$tpaApiDataModel = new TpaApiDataModel();
|
||||
|
||||
// echo $file_id;die();
|
||||
//deactivate old data
|
||||
$tpaApiDataModel->set('is_active', 0)->where('file_id', $file_id)->update();
|
||||
|
||||
//covert tpa data to our model data
|
||||
$mappedRows = [];
|
||||
|
||||
foreach ($records as $row) {
|
||||
|
||||
$mappedRows[] = [
|
||||
'file_id' => $file_id, // ← pass from controller
|
||||
'emp_code' => trim($row['priBenefEmpCode'] ?? ''),
|
||||
|
||||
'name' => trim($row['benefName'] ?? ''),
|
||||
'dob' => !empty($row['benefDOB'])
|
||||
? date('Y-m-d', strtotime(str_replace('/', '-', $row['benefDOB'])))
|
||||
: null,
|
||||
|
||||
'relation' => trim($row['relName'] ?? null),
|
||||
'gender' => strtoupper($row['benefSex'] ?? null),
|
||||
'self' => strtolower($row['relName'] ?? '') === 'self' ? 1 : 0,
|
||||
|
||||
'tpa_id' => trim($row['benefMediAssistID'] ?? null),
|
||||
'age' => is_numeric($row['benefAge'] ?? null)
|
||||
? (int) $row['benefAge']
|
||||
: null,
|
||||
|
||||
'is_active' => 1,
|
||||
'created_by' => $file_info[0]['created_by'] ?? null,
|
||||
];
|
||||
}
|
||||
// log_message('error','COUNT' . count($mappedRows));
|
||||
// print_rr($mappedRows);//die();
|
||||
$result = $tpaApiDataModel->insertBatchWithChunkLog($mappedRows,500,('FILE_ID_'.$file_id));
|
||||
// unlink($file_array['json_file_path']); // delete temp json file
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
239
app/Helpers/TPADataCompareHelper.php
Normal file
239
app/Helpers/TPADataCompareHelper.php
Normal file
@ -0,0 +1,239 @@
|
||||
<?php
|
||||
|
||||
namespace App\Helpers;
|
||||
|
||||
use DateTime;
|
||||
|
||||
class TPADataCompareHelper
|
||||
{
|
||||
/** Common fields to compare */
|
||||
public const COMPARE_FIELDS = ['name', 'dob', 'gender', 'relation'];
|
||||
|
||||
/* =========================
|
||||
Normalization utilities
|
||||
========================= */
|
||||
|
||||
public static function parseDateToYmd(?string $date): ?string
|
||||
{
|
||||
if (empty($date)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
$formats = [
|
||||
'd/m/Y H:i:s',
|
||||
'd/m/Y',
|
||||
'Y-m-d H:i:s',
|
||||
'Y-m-d',
|
||||
'd-m-Y',
|
||||
];
|
||||
|
||||
foreach ($formats as $format) {
|
||||
$dt = DateTime::createFromFormat($format, $date);
|
||||
if ($dt !== false) {
|
||||
return $dt->format('Y-m-d');
|
||||
}
|
||||
}
|
||||
|
||||
$ts = strtotime($date);
|
||||
return $ts ? date('Y-m-d', $ts) : null;
|
||||
}
|
||||
|
||||
public static function normalizeName(?string $name): ?string
|
||||
{
|
||||
if ($name === null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
$name = trim($name);
|
||||
// $name = preg_replace('/\.+/', ' ', $name);
|
||||
// $name = preg_replace('/\s+/', ' ', $name);
|
||||
return mb_strtolower($name);
|
||||
}
|
||||
|
||||
public static function normalizeEmpId(?string $empId): ?string
|
||||
{
|
||||
if ($empId === null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
$empId = trim($empId);
|
||||
$empId = preg_replace('/[\s\-\/]/', '', $empId);
|
||||
return $empId === '' ? null : $empId;
|
||||
}
|
||||
|
||||
public static function normalizeGender(?string $gender): ?string
|
||||
{
|
||||
if ($gender === null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
$g = strtoupper(trim($gender));
|
||||
return in_array($g, ['M', 'MALE']) ? 'M'
|
||||
: (in_array($g, ['F', 'FEMALE']) ? 'F' : $g);
|
||||
}
|
||||
|
||||
public static function normalizeRelation(?string $relation): ?string
|
||||
{
|
||||
return $relation === null ? null : strtoupper(trim($relation));
|
||||
}
|
||||
|
||||
/* =========================
|
||||
Core normalization
|
||||
========================= */
|
||||
|
||||
public static function normalizeEmployees(
|
||||
array $rows,
|
||||
array $keyMap,
|
||||
?string $pkKey = null
|
||||
): array {
|
||||
$normalized = [];
|
||||
// print_r($rows);die();
|
||||
foreach ($rows as $index => $row) {
|
||||
// echo "Normalizing record #" . ($index + 1) . "\n";
|
||||
$item = [
|
||||
'pk' => $pkKey && isset($row[$pkKey]) ? $row[$pkKey] : null,
|
||||
'__raw' => $row,
|
||||
];
|
||||
|
||||
foreach ($keyMap as $canonical => $possibleKeys) {
|
||||
$value = null;
|
||||
|
||||
foreach ($possibleKeys as $key) {
|
||||
if (isset($row[$key]) && $row[$key] !== '') {
|
||||
$value = $row[$key];
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
switch ($canonical) {
|
||||
case 'emp_id':
|
||||
$item['emp_id'] = self::normalizeEmpId($value);
|
||||
break;
|
||||
case 'name':
|
||||
$item['name'] = self::normalizeName($value);
|
||||
break;
|
||||
case 'dob':
|
||||
$item['dob'] = self::parseDateToYmd($value);
|
||||
break;
|
||||
case 'gender':
|
||||
$item['gender'] = self::normalizeGender($value);
|
||||
break;
|
||||
case 'relation':
|
||||
$item['relation'] = self::normalizeRelation($value);
|
||||
break;
|
||||
default:
|
||||
$item[$canonical] = $value;
|
||||
}
|
||||
}
|
||||
|
||||
if (!empty($item['pk'])) {
|
||||
$normalized[$item['pk']] = $item;
|
||||
} else {
|
||||
$normalized['__no_emp_' . $index] = $item;
|
||||
}
|
||||
|
||||
// echo "Normalizing record #" . ($index + 1) . "\n";
|
||||
}
|
||||
|
||||
return $normalized;
|
||||
}
|
||||
|
||||
/* =========================
|
||||
Main comparison
|
||||
========================= */
|
||||
|
||||
public static function compareDbVsMediassist(
|
||||
array $dbData,
|
||||
array $mediassistAPIdata
|
||||
): array {
|
||||
// echo count($dbData) . " records in DB data\n";die();
|
||||
$dbKeyMap = [
|
||||
'emp_id' => ['emp_code'],
|
||||
'name' => ['name'],
|
||||
'dob' => ['dob'],
|
||||
'gender' => ['gender'],
|
||||
'relation' => ['relationship'],
|
||||
];
|
||||
|
||||
$mediassistKeyMap = [
|
||||
'emp_id' => ['priBenefEmpCode'],
|
||||
'name' => ['benefName', 'priBeneficiaryName'],
|
||||
'dob' => ['benefDOB'],
|
||||
'gender' => ['benefSex'],
|
||||
'relation' => ['relName'],
|
||||
];
|
||||
|
||||
$A = self::normalizeEmployees($dbData, $dbKeyMap, 'id');
|
||||
|
||||
$B = self::normalizeEmployees($mediassistAPIdata, $mediassistKeyMap);
|
||||
dd($B);
|
||||
$report = [
|
||||
'summary' => [
|
||||
'total_db' => count($A),
|
||||
'total_mediassist' => count($B),
|
||||
'matched' => 0,
|
||||
'mismatched' => 0,
|
||||
'only_in_db' => 0,
|
||||
'only_in_mediassist' => 0,
|
||||
],
|
||||
'exact_matches' => [],
|
||||
'field_mismatches' => [],
|
||||
'only_in_db' => [],
|
||||
'only_in_mediassist' => [],
|
||||
];
|
||||
|
||||
foreach ($A as $empId => $rowA) {
|
||||
|
||||
if (!isset($B[$empId])) {
|
||||
$report['only_in_db'][] = [
|
||||
'pk' => $rowA['pk'],
|
||||
'emp_id' => $empId,
|
||||
'data' => $rowA,
|
||||
];
|
||||
continue;
|
||||
}
|
||||
|
||||
$rowB = $B[$empId];
|
||||
$diff = [];
|
||||
|
||||
foreach (self::COMPARE_FIELDS as $field) {
|
||||
if (($rowA[$field] ?? null) !== ($rowB[$field] ?? null)) {
|
||||
$diff[$field] = [
|
||||
'db' => $rowA[$field] ?? null,
|
||||
'mediassist' => $rowB[$field] ?? null,
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
if (empty($diff)) {
|
||||
$report['exact_matches'][] = [
|
||||
'pk' => $rowA['pk'],
|
||||
'emp_id' => $empId,
|
||||
];
|
||||
$report['summary']['matched']++;
|
||||
} else {
|
||||
$report['field_mismatches'][] = [
|
||||
'pk' => $rowA['pk'],
|
||||
'emp_id' => $empId,
|
||||
'differences' => $diff,
|
||||
];
|
||||
$report['summary']['mismatched']++;
|
||||
}
|
||||
|
||||
unset($B[$empId]);
|
||||
}
|
||||
|
||||
foreach ($B as $empId => $rowB) {
|
||||
$report['only_in_mediassist'][] = [
|
||||
'pk' => null,
|
||||
'emp_id' => $empId,
|
||||
'data' => $rowB,
|
||||
];
|
||||
}
|
||||
|
||||
$report['summary']['only_in_db'] = count($report['only_in_db']);
|
||||
$report['summary']['only_in_mediassist'] = count($report['only_in_mediassist']);
|
||||
|
||||
return $report;
|
||||
}
|
||||
}
|
||||
363
app/Helpers/TPADataCompareHelper2.php
Normal file
363
app/Helpers/TPADataCompareHelper2.php
Normal file
@ -0,0 +1,363 @@
|
||||
<?php
|
||||
|
||||
namespace App\Helpers;
|
||||
|
||||
use DateTime;
|
||||
|
||||
class TPADataCompareHelper2
|
||||
{
|
||||
// fields compared by default
|
||||
public const COMPARE_FIELDS = ['name', 'dob', 'gender', 'relation'];
|
||||
|
||||
// candidate unique id fields from MediAssist / Excel — prefer these when present
|
||||
public const MEDIASSIST_UNIQUE_IDS = [
|
||||
'benefMediAssistID',
|
||||
'benefSlNoasperInsurer',
|
||||
'benefNIAPersonID'
|
||||
];
|
||||
|
||||
/* -------------------------
|
||||
Normalization utilities
|
||||
------------------------- */
|
||||
|
||||
public static function parseDateToYmd(?string $date): ?string
|
||||
{
|
||||
if ($date === null || $date === '') return null;
|
||||
$formats = ['d/m/Y H:i:s','d/m/Y','Y-m-d H:i:s','Y-m-d','d-m-Y','d-m-Y H:i:s'];
|
||||
foreach ($formats as $f) {
|
||||
$dt = DateTime::createFromFormat($f, $date);
|
||||
if ($dt !== false) return $dt->format('Y-m-d');
|
||||
}
|
||||
$ts = strtotime($date);
|
||||
return $ts ? date('Y-m-d', $ts) : null;
|
||||
}
|
||||
|
||||
public static function normalizeName(?string $name): ?string
|
||||
{
|
||||
if ($name === null) return null;
|
||||
$s = trim($name);
|
||||
// $s = preg_replace('/\.+/', ' ', $s);
|
||||
// $s = preg_replace('/\s+/', ' ', $s);
|
||||
return mb_strtolower($s);
|
||||
}
|
||||
|
||||
public static function normalizeEmpId(?string $id): ?string
|
||||
{
|
||||
if ($id === null) return null;
|
||||
$s = trim($id);
|
||||
// $s = preg_replace('/[\s\-\/]/', '', $s);
|
||||
// $s = preg_replace('/[^\p{L}\p{N}]/u', '', $s);
|
||||
return $s === '' ? null : $s;
|
||||
}
|
||||
|
||||
public static function normalizeGender(?string $g): ?string
|
||||
{
|
||||
if ($g === null) return null;
|
||||
$u = strtoupper(trim($g));
|
||||
if (in_array($u, ['M','MALE'])) return 'M';
|
||||
if (in_array($u, ['F','FEMALE'])) return 'F';
|
||||
return $u;
|
||||
}
|
||||
|
||||
public static function normalizeRelation(?string $r): ?string
|
||||
{
|
||||
if ($r === null) return null;
|
||||
return strtoupper(trim($r));
|
||||
}
|
||||
|
||||
/* -------------------------------------
|
||||
Normalize rows -> canonical structure
|
||||
Returns array of items (not keyed), each item contains:
|
||||
- 'pk' (DB primary key or null)
|
||||
- 'emp_id' (employee code)
|
||||
- 'name','dob','gender','relation' (normalized)
|
||||
- '__raw' original row
|
||||
- '__rowid' generated unique row id for API rows (if needed)
|
||||
------------------------------------- */
|
||||
public static function normalizeRows(array $rows, array $keyMap, ?string $pkKey = null): array
|
||||
{
|
||||
$out = [];
|
||||
$idx = 0;
|
||||
foreach ($rows as $r) {
|
||||
$item = [
|
||||
'pk' => ($pkKey && isset($r[$pkKey])) ? $r[$pkKey] : null,
|
||||
'__raw' => $r,
|
||||
'__rowid' => null,
|
||||
];
|
||||
|
||||
// fill canonical fields
|
||||
foreach ($keyMap as $canonical => $candidates) {
|
||||
$val = null;
|
||||
foreach ($candidates as $k) {
|
||||
if (is_array($r) && array_key_exists($k, $r) && $r[$k] !== '') {
|
||||
$val = $r[$k];
|
||||
break;
|
||||
}
|
||||
}
|
||||
switch ($canonical) {
|
||||
case 'emp_id': $item['emp_id'] = self::normalizeEmpId((string)$val); break;
|
||||
case 'name': $item['name'] = self::normalizeName((string)$val); break;
|
||||
case 'dob': $item['dob'] = self::parseDateToYmd((string)$val); break;
|
||||
case 'gender': $item['gender'] = self::normalizeGender((string)$val); break;
|
||||
case 'relation': $item['relation'] = self::normalizeRelation((string)$val); break;
|
||||
default: $item[$canonical] = $val;
|
||||
}
|
||||
}
|
||||
|
||||
// create a stable row id (for API rows without PK) so they remain unique
|
||||
$item['__rowid'] = $item['pk'] ?? ('api_row_' . $idx++);
|
||||
|
||||
$out[] = $item;
|
||||
}
|
||||
|
||||
return $out;
|
||||
}
|
||||
|
||||
/* -------------------------
|
||||
Build grouped index by emp_code:
|
||||
returns [ emp_code => [ item1, item2, ... ] ]
|
||||
Use '__no_emp' key for rows without emp_id
|
||||
------------------------- */
|
||||
public static function groupByEmpCode(array $items): array
|
||||
{
|
||||
$groups = [];
|
||||
foreach ($items as $it) {
|
||||
$key = $it['emp_id'] ?? '__no_emp';
|
||||
if ($key === null || $key === '') $key = '__no_emp';
|
||||
$groups[$key][] = $it;
|
||||
}
|
||||
return $groups;
|
||||
}
|
||||
|
||||
/* -------------------------
|
||||
Find a match for a DB row inside a mediassist group (array of items).
|
||||
Matching priority:
|
||||
1) unique ids present in mediassist (if db raw has any of those too)
|
||||
2) exact name + dob
|
||||
3) exact name + relation
|
||||
4) fuzzy name + dob (threshold %)
|
||||
If found, returns index of matched item in $group array; else null.
|
||||
Note: $group is passed by reference so matched item can be removed by caller.
|
||||
------------------------- */
|
||||
public static function findMatchInGroup(array $dbRow, array $group, int $fuzzyThreshold = 85): ?int
|
||||
{
|
||||
// 1) match on unique ids if DB row contains any of those (rare)
|
||||
// foreach (self::MEDIASSIST_UNIQUE_IDS as $uidField) {
|
||||
// $dbVal = $dbRow['__raw'][$uidField] ?? null;
|
||||
// if ($dbVal) {
|
||||
// foreach ($group as $i => $gItem) {
|
||||
// $gVal = $gItem['__raw'][$uidField] ?? null;
|
||||
// if ($gVal && (string)$gVal === (string)$dbVal) return $i;
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
|
||||
// 2) match exact name + dob (both normalized)
|
||||
if (!empty($dbRow['name']) && !empty($dbRow['dob'])) {
|
||||
foreach ($group as $i => $gItem) {
|
||||
if (!empty($gItem['name']) && !empty($gItem['dob'])
|
||||
&& $dbRow['name'] === $gItem['name']
|
||||
&& $dbRow['dob'] === $gItem['dob']) {
|
||||
return $i;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 3) match exact name + relation (if dob missing or unreliable)
|
||||
if (!empty($dbRow['name']) && !empty($dbRow['relation'])) {
|
||||
foreach ($group as $i => $gItem) {
|
||||
if (!empty($gItem['name']) && !empty($gItem['relation'])
|
||||
&& $dbRow['name'] === $gItem['name']
|
||||
&& $dbRow['relation'] === $gItem['relation']) {
|
||||
return $i;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 4) fuzzy name + dob (if dob present)
|
||||
if ($fuzzyThreshold > 0 && !empty($dbRow['name'])) {
|
||||
foreach ($group as $i => $gItem) {
|
||||
if (empty($gItem['name'])) continue;
|
||||
$score = 0.0;
|
||||
similar_text($dbRow['name'], $gItem['name'], $score); // percent
|
||||
$dobMatches = (!empty($dbRow['dob']) && !empty($gItem['dob']) && $dbRow['dob'] === $gItem['dob']);
|
||||
// require DOB match OR higher threshold if dob missing
|
||||
if ($dobMatches && $score >= max(60, $fuzzyThreshold - 10)) { // slightly relaxed
|
||||
return $i;
|
||||
}
|
||||
if (!$dobMatches && $score >= $fuzzyThreshold) {
|
||||
// if names are very similar even w/o dob, accept (cautious)
|
||||
return $i;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// no match
|
||||
return null;
|
||||
}
|
||||
|
||||
/* -------------------------
|
||||
Main comparison routine (grouped matching)
|
||||
Options:
|
||||
- fuzzy_threshold (default 85)
|
||||
- try_global_search_if_emp_missing (bool, default false)
|
||||
------------------------- */
|
||||
public static function compareDbVsMediassistGrouped(
|
||||
array $dbRows,
|
||||
array $mediassistAPIdata,
|
||||
string $dbPkKey = 'id',
|
||||
array $options = [],
|
||||
array $dbKeyMap = null,
|
||||
array $mediassistKeyMap = null,
|
||||
): array {
|
||||
|
||||
|
||||
$dbKeyMap = [
|
||||
'emp_id' => ['emp_code'],
|
||||
'name' => ['name'],
|
||||
'dob' => ['dob'],
|
||||
'gender' => ['gender'],
|
||||
'relation' => ['relationship'],
|
||||
];
|
||||
|
||||
$mediassistKeyMap = [
|
||||
'emp_id' => ['priBenefEmpCode'],
|
||||
'name' => ['benefName', 'priBeneficiaryName'],
|
||||
'dob' => ['benefDOB'],
|
||||
'gender' => ['benefSex'],
|
||||
'relation' => ['relName'],
|
||||
];
|
||||
|
||||
|
||||
$fuzzyThreshold = $options['fuzzy_threshold'] ?? 85;
|
||||
$tryGlobalSearch = $options['try_global_search_if_emp_missing'] ?? false;
|
||||
|
||||
// normalize to lists
|
||||
$normDb = self::normalizeRows($dbRows, $dbKeyMap, $dbPkKey);
|
||||
$normApi = self::normalizeRows($mediassistAPIdata, $mediassistKeyMap, null);
|
||||
// dd($normApi);
|
||||
// group by emp_code
|
||||
$dbGroups = self::groupByEmpCode($normDb);
|
||||
$apiGroups = self::groupByEmpCode($normApi);
|
||||
|
||||
// prepare report
|
||||
$report = [
|
||||
'summary' => [
|
||||
'total_db' => count($normDb),
|
||||
'total_mediassist' => count($normApi),
|
||||
'matched' => 0,
|
||||
'mismatched' => 0,
|
||||
'only_in_db' => 0,
|
||||
'only_in_mediassist' => 0,
|
||||
],
|
||||
'exact_matches' => [],
|
||||
'field_mismatches' => [],
|
||||
'only_in_db' => [],
|
||||
'only_in_mediassist' => [],
|
||||
];
|
||||
|
||||
// For each db group (emp_code), try match each dbRow with api group entries
|
||||
foreach ($dbGroups as $empCode => $dbItems) {
|
||||
$apiItems = $apiGroups[$empCode] ?? [];
|
||||
|
||||
// We'll mutate a local copy of apiItems to mark matched items (so we can remove)
|
||||
$remainingApi = $apiItems;
|
||||
|
||||
foreach ($dbItems as $dbRow) {
|
||||
$matchedIndex = null;
|
||||
if (!empty($empCode) && !empty($remainingApi)) {
|
||||
$matchedIndex = self::findMatchInGroup($dbRow, $remainingApi, $fuzzyThreshold);
|
||||
}
|
||||
|
||||
// If empCode not present in API and global search allowed, try searching all API rows (costly)
|
||||
if ($matchedIndex === null && $tryGlobalSearch && empty($apiItems)) {
|
||||
// attempt global search across all api groups (can be heavy)
|
||||
foreach ($apiGroups as $gKey => $gList) {
|
||||
$mi = self::findMatchInGroup($dbRow, $gList, $fuzzyThreshold);
|
||||
if ($mi !== null) {
|
||||
// found in another group
|
||||
$matchedIndex = $mi;
|
||||
// use that group as remainingApi reference so we can remove matched later
|
||||
$remainingApi = $apiGroups[$gKey];
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if ($matchedIndex !== null) {
|
||||
// found match — use that api row
|
||||
$apiMatch = $remainingApi[$matchedIndex];
|
||||
|
||||
// compare fields
|
||||
$diff = [];
|
||||
foreach (self::COMPARE_FIELDS as $f) {
|
||||
$vDb = $dbRow[$f] ?? null;
|
||||
$vApi = $apiMatch[$f] ?? null;
|
||||
if ($vDb !== $vApi) {
|
||||
$diff[$f] = ['db' => $vDb, 'mediassist' => $vApi];
|
||||
}
|
||||
}
|
||||
|
||||
if (empty($diff)) {
|
||||
$report['exact_matches'][] = [
|
||||
'pk' => $dbRow['pk'],
|
||||
'emp_id' => $dbRow['emp_id'],
|
||||
'db_rowid' => $dbRow['__rowid'],
|
||||
'api_rowid' => $apiMatch['__rowid'],
|
||||
];
|
||||
$report['summary']['matched']++;
|
||||
} else {
|
||||
$report['field_mismatches'][] = [
|
||||
'pk' => $dbRow['pk'],
|
||||
'emp_id' => $dbRow['emp_id'],
|
||||
'db_rowid' => $dbRow['__rowid'],
|
||||
'api_rowid' => $apiMatch['__rowid'],
|
||||
'differences' => $diff
|
||||
];
|
||||
$report['summary']['mismatched']++;
|
||||
}
|
||||
|
||||
// remove matched api item from remainingApi (so it's not matched again)
|
||||
array_splice($remainingApi, $matchedIndex, 1);
|
||||
} else {
|
||||
// no match found for this dbRow inside api group => only_in_db
|
||||
$report['only_in_db'][] = [
|
||||
'pk' => $dbRow['pk'],
|
||||
'emp_id' => $dbRow['emp_id'],
|
||||
'db_rowid' => $dbRow['__rowid'],
|
||||
'data' => $dbRow
|
||||
];
|
||||
}
|
||||
} // end each dbRow
|
||||
|
||||
// after processing all dbRows in this empCode group, any remainingApi are only_in_mediassist
|
||||
foreach ($remainingApi as $leftApi) {
|
||||
$report['only_in_mediassist'][] = [
|
||||
'pk' => null,
|
||||
'emp_id' => $leftApi['emp_id'],
|
||||
'api_rowid' => $leftApi['__rowid'],
|
||||
'data' => $leftApi
|
||||
];
|
||||
}
|
||||
} // end each emp group
|
||||
|
||||
// Also handle any api groups whose emp_code does not exist in DB at all
|
||||
foreach ($apiGroups as $empCode => $apiItems) {
|
||||
if (isset($dbGroups[$empCode])) continue; // already handled
|
||||
foreach ($apiItems as $ai) {
|
||||
$report['only_in_mediassist'][] = [
|
||||
'pk' => null,
|
||||
'emp_id' => $ai['emp_id'],
|
||||
'api_rowid' => $ai['__rowid'],
|
||||
'data' => $ai
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
// summary counts
|
||||
$report['summary']['only_in_db'] = count($report['only_in_db']);
|
||||
$report['summary']['only_in_mediassist'] = count($report['only_in_mediassist']);
|
||||
|
||||
return $report;
|
||||
}
|
||||
}
|
||||
108
app/Models/TpaApiDataModel.php
Normal file
108
app/Models/TpaApiDataModel.php
Normal file
@ -0,0 +1,108 @@
|
||||
<?php
|
||||
|
||||
namespace App\Models;
|
||||
|
||||
use CodeIgniter\Model;
|
||||
|
||||
class TpaApiDataModel extends Model
|
||||
{
|
||||
protected $table = 'tpa_api_data';
|
||||
protected $primaryKey = 'id';
|
||||
protected $useAutoIncrement = true;
|
||||
|
||||
protected $returnType = 'array';
|
||||
protected $useSoftDeletes = false;
|
||||
|
||||
protected $allowedFields = [
|
||||
'file_id',
|
||||
'emp_code',
|
||||
'name',
|
||||
'dob',
|
||||
'relation',
|
||||
'gender',
|
||||
'self',
|
||||
'tpa_id',
|
||||
'age',
|
||||
'is_active',
|
||||
'desc',
|
||||
'created_by'
|
||||
];
|
||||
|
||||
// protected $useTimestamps = true;
|
||||
protected $createdField = 'create_at';
|
||||
protected $updatedField = null;
|
||||
|
||||
// protected $dateFormat = 'datetime';
|
||||
|
||||
// protected $validationRules = [
|
||||
// 'file_id' => 'required|integer',
|
||||
// 'temp_code' => 'required|string|max_length[50]',
|
||||
// 'tname' => 'required|string|max_length[150]',
|
||||
// 'tgender' => 'permit_empty|in_list[M,F,O]',
|
||||
// 'tage' => 'permit_empty|integer'
|
||||
// ];
|
||||
|
||||
protected $validationMessages = [];
|
||||
protected $skipValidation = false;
|
||||
|
||||
|
||||
|
||||
|
||||
public function insertBatchWithChunkLog( array $rows,int $chunkSize = 500,?string $logRef = null)
|
||||
{
|
||||
if (empty($rows)) {
|
||||
return [
|
||||
'status' => false,
|
||||
'inserted' => 0,
|
||||
'failed_batches' => 0,
|
||||
'message' => 'No data'
|
||||
];
|
||||
}
|
||||
|
||||
$db = \Config\Database::connect();
|
||||
$this->skipValidation(true);
|
||||
|
||||
$inserted = 0;
|
||||
$failedBatches = 0;
|
||||
$batchNo = 0;
|
||||
|
||||
foreach (array_chunk($rows, $chunkSize) as $chunk) {
|
||||
$batchNo++;
|
||||
|
||||
try {
|
||||
$db->transStart();
|
||||
|
||||
$this->insertBatch($chunk);
|
||||
|
||||
if ($db->transStatus() === false) {
|
||||
throw new \RuntimeException('DB transaction failed');
|
||||
}
|
||||
|
||||
$db->transComplete();
|
||||
$inserted += count($chunk);
|
||||
|
||||
} catch (\Throwable $e) {
|
||||
$db->transRollback();
|
||||
$failedBatches++;
|
||||
|
||||
// 🔴 Batch-level error log
|
||||
log_message('error', json_encode([
|
||||
'type' => 'BATCH_INSERT_FAILED',
|
||||
'table' => $this->table,
|
||||
'batch_no' => $batchNo,
|
||||
'batch_size'=> count($chunk),
|
||||
'ref' => $logRef,
|
||||
'error' => $e->getMessage()
|
||||
]));
|
||||
}
|
||||
}
|
||||
|
||||
return [
|
||||
'status' => $failedBatches === 0,
|
||||
'inserted' => $inserted,
|
||||
'failed_batches' => $failedBatches,
|
||||
'total_batches' => $batchNo
|
||||
];
|
||||
}
|
||||
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user