diff --git a/app/Controllers/PolicyRagController.php b/app/Controllers/PolicyRagController.php index fe413ba..304e551 100644 --- a/app/Controllers/PolicyRagController.php +++ b/app/Controllers/PolicyRagController.php @@ -66,7 +66,7 @@ class PolicyRagController extends ResourceController { $policyId = $data['policy_id']; - log_message('debug', 'PolicyRag readFileAndCalculateCommission started for policy_id: ' . $policyId); + $this->logRagStep('JOB_START', 'started', 'readFileAndCalculateCommission', ['policy_id' => $policyId]); $readDoc = $this->readPolicyDocViaRag($policyId); $calculateCommission = ['status' => 'failed']; @@ -75,21 +75,43 @@ class PolicyRagController extends ResourceController $extracted = $readDoc['data']; $policyData = $this->mapExtractedDataToPolicyFields($extracted); - log_message('debug', 'Policy RAG Update Payload: ' . json_encode($policyData)); + $this->logRagStep('DB_UPDATE', 'started', 'Updating policy from RAG data', [ + 'policy_id' => $policyId, + 'payload' => $policyData, + ]); $updateStatus = $this->PolicyModel->update($policyId, $policyData); if ($updateStatus) { - log_message('debug', 'Policy updated successfully via RAG for ID: ' . $policyId); + $this->logRagStep('DB_UPDATE', 'success', 'Policy updated from RAG data', ['policy_id' => $policyId]); + + $this->logRagStep('COMMISSION', 'started', 'Calculating commission', ['policy_id' => $policyId]); $policyController = new PolicyController(); $calculateCommission = $policyController->calculateCommission($policyId); + + $commissionStatus = $calculateCommission['status'] ?? 'failed'; + $this->logRagStep( + 'COMMISSION', + $commissionStatus === 'success' ? 'success' : 'failed', + 'Commission calculation finished', + ['policy_id' => $policyId, 'result' => $commissionStatus] + ); } else { - log_message('error', 'Failed to update policy via RAG for ID: ' . $policyId); + $this->logRagStep('DB_UPDATE', 'failed', 'Failed to update policy from RAG data', ['policy_id' => $policyId]); } } else { - log_message('error', 'readPolicyDocViaRag returned failure: ' . json_encode($readDoc)); + $this->logRagStep('JOB_READ', 'failed', 'readPolicyDocViaRag failed', [ + 'policy_id' => $policyId, + 'response' => $readDoc, + ]); } + $this->logRagStep('JOB_END', 'completed', 'readFileAndCalculateCommission finished', [ + 'policy_id' => $policyId, + 'readPolicyDocViaRag' => $readDoc['status'], + 'calculateCommission' => $calculateCommission['status'] ?? 'failed', + ]); + return [ 'readPolicyDocViaRag' => $readDoc['status'], 'calculateCommission' => $calculateCommission['status'] ?? 'failed', @@ -130,67 +152,126 @@ class PolicyRagController extends ResourceController { $fileId = null; + $this->logRagStep('READ_START', 'started', 'readPolicyDocViaRag', ['policy_id' => $policyId]); + try { $record = $this->PolicyModel->find((int) $policyId); if (!$record) { + $this->logRagStep('POLICY_LOOKUP', 'failed', 'Policy not found', ['policy_id' => $policyId]); return $this->formatReadResponse('failed', 'Policy not found'); } + $this->logRagStep('POLICY_LOOKUP', 'success', 'Policy found', [ + 'policy_id' => $policyId, + 'pdf_file' => $record['policy_pdf_file_name'] ?? null, + ]); + $uploadedPath = WRITEPATH . 'uploads/policy/policy_pdf/'; $pdfFilePath = $uploadedPath . $record['policy_pdf_file_name']; if (!file_exists($pdfFilePath)) { - log_message('info', "RAG read: PDF not found at {$pdfFilePath}"); + $this->logRagStep('PDF_CHECK', 'failed', 'PDF file not found', ['path' => $pdfFilePath]); return $this->formatReadResponse('failed', "File not found at {$pdfFilePath}"); } - $uploadResult = $this->uploadFileToRag($pdfFilePath); + $this->logRagStep('PDF_CHECK', 'success', 'PDF file found', ['path' => $pdfFilePath]); + + $this->logRagStep('UPLOAD', 'started', 'Uploading PDF to RAG API', ['path' => $pdfFilePath]); + $uploadResult = $this->uploadFileToRag($pdfFilePath, (int) $policyId); if ($uploadResult['status'] !== 'success') { return $this->formatReadResponse('failed', $uploadResult['message']); } $fileId = $uploadResult['data']['file_id'] ?? null; if (empty($fileId)) { + $this->logRagStep('UPLOAD', 'failed', 'RAG upload did not return file_id', ['policy_id' => $policyId]); return $this->formatReadResponse('failed', 'RAG upload did not return file_id'); } + $this->logRagStep('SAVE_RAG_FILE_ID', 'started', 'Saving rag_file_id', [ + 'policy_id' => $policyId, + 'file_id' => $fileId, + ]); + $updated = $this->PolicyModel->update((int) $policyId, ['rag_file_id' => $fileId]); if ($updated === false) { - log_message('error', "Failed to save rag_file_id={$fileId} for policy_id={$policyId}"); + $this->logRagStep('SAVE_RAG_FILE_ID', 'failed', 'Failed to save rag_file_id', [ + 'policy_id' => $policyId, + 'file_id' => $fileId, + ]); + } else { + $this->logRagStep('SAVE_RAG_FILE_ID', 'success', 'rag_file_id saved', [ + 'policy_id' => $policyId, + 'file_id' => $fileId, + ]); } - log_message('info', "RAG file uploaded. file_id={$fileId}, policy_id={$policyId}"); - $indexResult = $this->waitForFileIndexed($fileId); + $this->logRagStep('INDEX_POLL', 'started', 'Waiting for RAG file to be indexed', [ + 'policy_id' => $policyId, + 'file_id' => $fileId, + ]); + + $indexResult = $this->waitForFileIndexed($fileId, (int) $policyId); if ($indexResult['status'] !== 'success') { return $this->formatReadResponse('failed', $indexResult['message']); } - $chatResult = $this->chatWithRag($fileId, $this->getPolicyReadPrompt()); + $this->logRagStep('CHAT', 'started', 'Sending extraction prompt to RAG API', [ + 'policy_id' => $policyId, + 'file_id' => $fileId, + ]); + + $chatResult = $this->chatWithRag($fileId, $this->getPolicyReadPrompt(), (int) $policyId); if ($chatResult['status'] !== 'success') { return $this->formatReadResponse('failed', $chatResult['message']); } $answer = $chatResult['data']['answer'] ?? ''; - // dd($answer); - $finalData = $this->extractJsonFromText($answer); + $finalData = $this->extractJsonFromText($answer, (int) $policyId); if ($finalData === null) { + $this->logRagStep('JSON_EXTRACT', 'failed', 'Could not extract valid JSON from RAG answer', [ + 'policy_id' => $policyId, + 'file_id' => $fileId, + 'answer_snippet' => substr($answer, 0, 500), + ]); return $this->formatReadResponse('failed', 'Could not extract valid JSON from RAG answer'); } - log_message('info', 'RAG policy doc read successful for policy_id=' . $policyId); + $this->logRagStep('READ_COMPLETE', 'success', 'Policy document read successfully', [ + 'policy_id' => $policyId, + 'file_id' => $fileId, + ]); + return $this->formatReadResponse('success', 'Doc read success', $finalData); } catch (\Throwable $e) { - log_message('error', 'PolicyRag readPolicyDocViaRag error: ' . $e->getMessage()); + $this->logRagStep('READ_ERROR', 'failed', $e->getMessage(), [ + 'policy_id' => $policyId, + 'file_id' => $fileId, + 'line' => $e->getLine(), + ]); return $this->formatReadResponse('failed', 'Error: ' . $e->getMessage()); } finally { if (!empty($fileId)) { - $deleteResult = $this->deleteRagFile($fileId); + $this->logRagStep('DELETE', 'started', 'Deleting RAG file', [ + 'policy_id' => $policyId, + 'file_id' => $fileId, + ]); + + $deleteResult = $this->deleteRagFile($fileId, (int) $policyId); if ($deleteResult['status'] !== 'success') { - log_message('error', 'RAG file delete failed for file_id=' . $fileId . ': ' . ($deleteResult['message'] ?? '')); + $this->logRagStep('DELETE', 'failed', $deleteResult['message'] ?? 'RAG file delete failed', [ + 'policy_id' => $policyId, + 'file_id' => $fileId, + ]); } } + + $this->logRagStep('READ_END', 'completed', 'readPolicyDocViaRag finished', [ + 'policy_id' => $policyId, + 'file_id' => $fileId, + ]); } } @@ -344,7 +425,7 @@ class PolicyRagController extends ResourceController return $baseUrl; } - private function uploadFileToRag(string $pdfFilePath): array + private function uploadFileToRag(string $pdfFilePath, ?int $policyId = null): array { $url = $this->ragApiBaseUrl() . '/api/files/upload'; $cfile = new \CURLFile($pdfFilePath, 'application/pdf', basename($pdfFilePath)); @@ -363,24 +444,41 @@ class PolicyRagController extends ResourceController curl_close($ch); if ($curlError) { - log_message('error', 'RAG upload cURL error: ' . $curlError); + $this->logRagStep('UPLOAD', 'failed', 'cURL error: ' . $curlError, [ + 'policy_id' => $policyId, + 'url' => $url, + ]); return ['status' => 'failed', 'message' => 'RAG upload cURL error: ' . $curlError]; } if ($httpCode < 200 || $httpCode >= 300) { - log_message('error', "RAG upload HTTP {$httpCode}: " . substr((string) $response, 0, 500)); + $this->logRagStep('UPLOAD', 'failed', "HTTP {$httpCode}", [ + 'policy_id' => $policyId, + 'response' => substr((string) $response, 0, 500), + ]); return ['status' => 'failed', 'message' => "RAG upload failed with HTTP {$httpCode}"]; } $responseData = json_decode((string) $response, true); if (!is_array($responseData) || empty($responseData['success'])) { + $this->logRagStep('UPLOAD', 'failed', 'Invalid API response', [ + 'policy_id' => $policyId, + 'response' => substr((string) $response, 0, 500), + ]); return ['status' => 'failed', 'message' => 'RAG upload returned invalid response']; } + $this->logRagStep('UPLOAD', 'success', 'PDF uploaded to RAG API', [ + 'policy_id' => $policyId, + 'file_id' => $responseData['file_id'] ?? null, + 'filename' => $responseData['filename'] ?? null, + 'status' => $responseData['status'] ?? null, + ]); + return ['status' => 'success', 'data' => $responseData]; } - private function getFileStatus(string $fileId): array + private function getFileStatus(string $fileId, ?int $policyId = null): array { $url = $this->ragApiBaseUrl() . '/api/files/status?file_id=' . urlencode($fileId); @@ -393,40 +491,67 @@ class PolicyRagController extends ResourceController curl_close($ch); if ($curlError) { + $this->logRagStep('INDEX_POLL', 'failed', 'Status check cURL error: ' . $curlError, [ + 'policy_id' => $policyId, + 'file_id' => $fileId, + ]); return ['status' => 'failed', 'message' => 'RAG status cURL error: ' . $curlError]; } if ($httpCode < 200 || $httpCode >= 300) { + $this->logRagStep('INDEX_POLL', 'failed', "Status check HTTP {$httpCode}", [ + 'policy_id' => $policyId, + 'file_id' => $fileId, + ]); return ['status' => 'failed', 'message' => "RAG status check failed with HTTP {$httpCode}"]; } $responseData = json_decode((string) $response, true); if (!is_array($responseData) || empty($responseData['success'])) { + $this->logRagStep('INDEX_POLL', 'failed', 'Status check invalid response', [ + 'policy_id' => $policyId, + 'file_id' => $fileId, + ]); return ['status' => 'failed', 'message' => 'RAG status returned invalid response']; } return ['status' => 'success', 'data' => $responseData]; } - private function waitForFileIndexed(string $fileId): array + private function waitForFileIndexed(string $fileId, ?int $policyId = null): array { $interval = (int) (getenv('RAG_API_STATUS_POLL_INTERVAL') ?: 10); $maxAttempts = (int) (getenv('RAG_API_STATUS_MAX_ATTEMPTS') ?: 60); for ($attempt = 1; $attempt <= $maxAttempts; $attempt++) { - $statusResult = $this->getFileStatus($fileId); + $statusResult = $this->getFileStatus($fileId, $policyId); if ($statusResult['status'] !== 'success') { return $statusResult; } $fileStatus = strtolower((string) ($statusResult['data']['status'] ?? '')); - log_message('info', "RAG file status poll attempt {$attempt}/{$maxAttempts}: file_id={$fileId}, status={$fileStatus}"); + $this->logRagStep('INDEX_POLL', 'progress', "Poll attempt {$attempt}/{$maxAttempts}", [ + 'policy_id' => $policyId, + 'file_id' => $fileId, + 'status' => $fileStatus, + ]); if ($fileStatus === 'indexed') { + $this->logRagStep('INDEX_POLL', 'success', 'File indexed', [ + 'policy_id' => $policyId, + 'file_id' => $fileId, + 'chunk_count' => $statusResult['data']['chunk_count'] ?? null, + 'attempts' => $attempt, + ]); return $statusResult; } if (in_array($fileStatus, ['failed', 'error'], true)) { + $this->logRagStep('INDEX_POLL', 'failed', 'Indexing failed', [ + 'policy_id' => $policyId, + 'file_id' => $fileId, + 'status' => $fileStatus, + ]); return ['status' => 'failed', 'message' => 'RAG file indexing failed with status: ' . $fileStatus]; } @@ -435,10 +560,17 @@ class PolicyRagController extends ResourceController } } + $this->logRagStep('INDEX_POLL', 'failed', 'Indexing timed out', [ + 'policy_id' => $policyId, + 'file_id' => $fileId, + 'max_attempts' => $maxAttempts, + 'interval_sec' => $interval, + ]); + return ['status' => 'failed', 'message' => 'RAG file indexing timed out']; } - private function chatWithRag(string $fileId, string $question): array + private function chatWithRag(string $fileId, string $question, ?int $policyId = null): array { $url = $this->ragApiBaseUrl() . '/api/chat'; @@ -459,23 +591,41 @@ class PolicyRagController extends ResourceController curl_close($ch); if ($curlError) { + $this->logRagStep('CHAT', 'failed', 'cURL error: ' . $curlError, [ + 'policy_id' => $policyId, + 'file_id' => $fileId, + ]); return ['status' => 'failed', 'message' => 'RAG chat cURL error: ' . $curlError]; } if ($httpCode < 200 || $httpCode >= 300) { - log_message('error', "RAG chat HTTP {$httpCode}: " . substr((string) $response, 0, 500)); + $this->logRagStep('CHAT', 'failed', "HTTP {$httpCode}", [ + 'policy_id' => $policyId, + 'file_id' => $fileId, + 'response' => substr((string) $response, 0, 500), + ]); return ['status' => 'failed', 'message' => "RAG chat failed with HTTP {$httpCode}"]; } $responseData = json_decode((string) $response, true); if (!is_array($responseData) || empty($responseData['success'])) { + $this->logRagStep('CHAT', 'failed', 'Invalid API response', [ + 'policy_id' => $policyId, + 'file_id' => $fileId, + ]); return ['status' => 'failed', 'message' => 'RAG chat returned invalid response']; } + $this->logRagStep('CHAT', 'success', 'RAG chat response received', [ + 'policy_id' => $policyId, + 'file_id' => $fileId, + 'answer_length' => strlen((string) ($responseData['answer'] ?? '')), + ]); + return ['status' => 'success', 'data' => $responseData]; } - private function deleteRagFile(string $fileId): array + private function deleteRagFile(string $fileId, ?int $policyId = null): array { $url = $this->ragApiBaseUrl() . '/api/files/id/' . urlencode($fileId); @@ -489,24 +639,40 @@ class PolicyRagController extends ResourceController curl_close($ch); if ($curlError) { + $this->logRagStep('DELETE', 'failed', 'cURL error: ' . $curlError, [ + 'policy_id' => $policyId, + 'file_id' => $fileId, + ]); return ['status' => 'failed', 'message' => 'RAG delete cURL error: ' . $curlError]; } if ($httpCode < 200 || $httpCode >= 300) { - log_message('error', "RAG delete HTTP {$httpCode}: " . substr((string) $response, 0, 500)); + $this->logRagStep('DELETE', 'failed', "HTTP {$httpCode}", [ + 'policy_id' => $policyId, + 'file_id' => $fileId, + 'response' => substr((string) $response, 0, 500), + ]); return ['status' => 'failed', 'message' => "RAG delete failed with HTTP {$httpCode}"]; } $responseData = json_decode((string) $response, true); if (!is_array($responseData) || empty($responseData['success'])) { + $this->logRagStep('DELETE', 'failed', 'Invalid API response', [ + 'policy_id' => $policyId, + 'file_id' => $fileId, + ]); return ['status' => 'failed', 'message' => 'RAG delete returned invalid response']; } - log_message('info', 'RAG file deleted: file_id=' . $fileId); + $this->logRagStep('DELETE', 'success', 'RAG file deleted', [ + 'policy_id' => $policyId, + 'file_id' => $fileId, + ]); + return ['status' => 'success', 'data' => $responseData]; } - private function extractJsonFromText(string $text): ?array + private function extractJsonFromText(string $text, ?int $policyId = null): ?array { $jsonString = ''; @@ -525,7 +691,28 @@ class PolicyRagController extends ResourceController } $decoded = json_decode($jsonString, true); - return is_array($decoded) ? $decoded : null; + if (!is_array($decoded)) { + $this->logRagStep('JSON_EXTRACT', 'failed', 'JSON decode failed', [ + 'policy_id' => $policyId, + 'json_snippet' => substr($jsonString, 0, 500), + ]); + return null; + } + + $this->logRagStep('JSON_EXTRACT', 'success', 'JSON extracted from RAG answer', [ + 'policy_id' => $policyId, + 'field_keys' => array_keys($decoded), + ]); + + return $decoded; + } + + private function logRagStep(string $step, string $status, string $message, array $context = []): void + { + $level = in_array($status, ['failed', 'error'], true) ? 'error' : 'info'; + $contextJson = $context ? ' | ' . json_encode($context, JSON_UNESCAPED_SLASHES) : ''; + + log_message($level, "[PolicyRag][{$step}][{$status}] {$message}{$contextJson}"); } private function formatReadResponse(string $status, string $message, ?array $data = null): array