merge: integrate wren-dev govdoc fixes

fastapi_modules/fastapi_leaudit/services/impl/govdocServiceImpl.py

@@ -6,6 +6,7 @@ import hashlib
 import json
 import mimetypes
 import time
+import uuid
 from dataclasses import dataclass
 from datetime import datetime
 from pathlib import Path
@@ -26,6 +27,7 @@ from fastapi_modules.fastapi_leaudit.govdoc_engine.parser.docx_parser import par
 from fastapi_modules.fastapi_leaudit.govdoc_engine.reporter.html_paragraph import paragraphs_to_html
 from fastapi_modules.fastapi_leaudit.models import LeauditDocument, LeauditDocumentFile
 from fastapi_modules.fastapi_leaudit.services import IGovdocService, IOssService
+from fastapi_modules.fastapi_leaudit.services.impl.documentServiceImpl import _find_latest_version_candidate
 from fastapi_modules.fastapi_leaudit.services.impl.ossServiceImpl import OssServiceImpl
 
 
@@ -35,6 +37,11 @@ class _GovdocDocumentRow:
     region: str
     processingStatus: str
     currentRunId: int | None
+    versionGroupKey: str | None
+    versionNo: int
+    rootVersionId: int | None
+    previousVersionId: int | None
+    isLatestVersion: bool
     createdAt: Any
     updatedAt: Any
     fileId: int
@@ -49,6 +56,7 @@ class _GovdocDocumentRow:
     passedCount: int | None
     failedCount: int | None
     skippedCount: int | None
+    resultSummaryJson: Any
     hasHtmlReport: bool
     hasDocxReport: bool
 
@@ -98,6 +106,29 @@ class GovdocServiceImpl(IGovdocService):
             await self._ensureGovdocSchema(session)
             currentUser = await self._getCurrentUserContext(createdBy)
             resolvedRegion = self._resolve_upload_region(currentUser, normalizedRegion)
+            previousVersionId: int | None = None
+            rootVersionId: int | None = None
+            versionGroupKey: str | None = None
+            versionNo = 1
+
+            latestCandidate = await self._find_govdoc_latest_version_candidate(
+                session,
+                typeId=typeId,
+                region=resolvedRegion,
+                normalizedName=normalizedName,
+                fileExt=fileExt,
+            )
+
+            if latestCandidate:
+                previousVersionId = int(latestCandidate["document_id"])
+                rootVersionId = int(latestCandidate["root_version_id"] or latestCandidate["document_id"])
+                versionGroupKey = str(latestCandidate["version_group_key"] or "")
+                versionNo = int(latestCandidate["version_no"] or 1) + 1
+                previousDocument = await session.get(LeauditDocument, previousVersionId)
+                if previousDocument is not None:
+                    previousDocument.isLatestVersion = False
+            else:
+                versionGroupKey = uuid.uuid4().hex
 
             document = await LeauditDocument.create_new(
                 session,
@@ -107,22 +138,23 @@ class GovdocServiceImpl(IGovdocService):
                 region=resolvedRegion,
                 processingStatus="waiting",
                 currentRunId=None,
-                versionGroupKey=None,
+                versionGroupKey=versionGroupKey,
-                versionNo=1,
+                versionNo=versionNo,
-                previousVersionId=None,
+                previousVersionId=previousVersionId,
-                rootVersionId=None,
+                rootVersionId=rootVersionId,
                 isLatestVersion=True,
                 normalizedName=normalizedName,
                 reviewScope="govdoc",
             )
-            document.rootVersionId = document.Id
+            if document.rootVersionId is None:
+                document.rootVersionId = document.Id
             await session.flush()
 
             objectKey = OssPathUtils.BuildBusinessDocKey(
                 Region=resolvedRegion,
                 TypeCode="govdoc",
                 DocumentId=document.Id,
-                Version="v1",
+                Version=f"v{document.versionNo or 1}",
                 FileRole="original",
                 FileName=fileName,
                 Year=uploadedAt.year,
@@ -224,6 +256,7 @@ class GovdocServiceImpl(IGovdocService):
             "f.is_active = true",
             "f.file_role = 'original'",
             "COALESCE(d.engine_type, 'leaudit') = 'govdoc'",
+            "COALESCE(d.is_latest_version, true) = true",
         ]
         filters.extend(
             self._buildDocumentScopeFilters(
@@ -269,6 +302,11 @@ class GovdocServiceImpl(IGovdocService):
                             COALESCE(d.region, 'default') AS region,
                             COALESCE(d.processing_status, 'waiting') AS processing_status,
                             d.current_run_id,
+                            d.version_group_key,
+                            COALESCE(d.version_no, 1) AS version_no,
+                            d.root_version_id,
+                            d.previous_version_id,
+                            COALESCE(d.is_latest_version, true) AS is_latest_version,
                             d.created_at,
                             d.updated_at,
                             f.id AS file_id,
@@ -283,6 +321,8 @@ class GovdocServiceImpl(IGovdocService):
                             gr.passed_count,
                             gr.failed_count,
                             gr.skipped_count,
+                            gr.result_summary_json,
+                            COALESCE(vc.total_versions, 1) AS total_versions,
                             EXISTS(
                                 SELECT 1
                                 FROM govdoc_report_artifacts gra
@@ -305,6 +345,15 @@ class GovdocServiceImpl(IGovdocService):
                          AND f.deleted_at IS NULL
                         LEFT JOIN govdoc_runs gr
                           ON gr.id = d.current_run_id
+                        LEFT JOIN (
+                            SELECT version_group_key, COUNT(*) AS total_versions
+                            FROM leaudit_documents
+                            WHERE deleted_at IS NULL
+                              AND COALESCE(engine_type, 'leaudit') = 'govdoc'
+                              AND COALESCE(version_group_key, '') <> ''
+                            GROUP BY version_group_key
+                        ) vc
+                          ON vc.version_group_key = d.version_group_key
                         WHERE {whereClause}
                         ORDER BY d.created_at DESC
                         LIMIT :limit OFFSET :offset
@@ -336,43 +385,97 @@ class GovdocServiceImpl(IGovdocService):
                 ).scalar_one()
             )
 
+            history_by_group: dict[str, list[dict[str, Any]]] = {}
+            total_versions_by_group = {
+                str(row["version_group_key"]): int(row.get("total_versions") or 1)
+                for row in rows
+                if row.get("version_group_key")
+            }
+            group_keys = [str(row["version_group_key"]) for row in rows if row.get("version_group_key")]
+            if group_keys:
+                history_rows = (
+                    await session.execute(
+                        text(
+                            """
+                            SELECT
+                                d.id AS document_id,
+                                COALESCE(d.region, 'default') AS region,
+                                COALESCE(d.processing_status, 'waiting') AS processing_status,
+                                d.current_run_id,
+                                d.version_group_key,
+                                COALESCE(d.version_no, 1) AS version_no,
+                                d.root_version_id,
+                                d.previous_version_id,
+                                COALESCE(d.is_latest_version, false) AS is_latest_version,
+                                d.created_at,
+                                d.updated_at,
+                                f.id AS file_id,
+                                f.file_name,
+                                f.file_ext,
+                                f.mime_type,
+                                f.file_size,
+                                f.oss_url,
+                                f.created_by,
+                                gr.result_status,
+                                gr.total_score,
+                                gr.passed_count,
+                                gr.failed_count,
+                                gr.skipped_count,
+                                gr.result_summary_json,
+                                EXISTS(
+                                    SELECT 1
+                                    FROM govdoc_report_artifacts gra
+                                    WHERE gra.run_id = d.current_run_id
+                                      AND gra.artifact_type = 'html_report'
+                                      AND gra.deleted_at IS NULL
+                                ) AS has_html_report,
+                                EXISTS(
+                                    SELECT 1
+                                    FROM govdoc_report_artifacts gra
+                                    WHERE gra.run_id = d.current_run_id
+                                      AND gra.artifact_type = 'annotated_docx'
+                                      AND gra.deleted_at IS NULL
+                                ) AS has_docx_report
+                            FROM leaudit_documents d
+                            JOIN leaudit_document_files f
+                              ON f.document_id = d.id
+                             AND f.is_active = true
+                             AND f.file_role = 'original'
+                             AND f.deleted_at IS NULL
+                            LEFT JOIN govdoc_runs gr
+                              ON gr.id = d.current_run_id
+                            WHERE d.deleted_at IS NULL
+                              AND COALESCE(d.engine_type, 'leaudit') = 'govdoc'
+                              AND d.version_group_key = ANY(:group_keys)
+                              AND COALESCE(d.is_latest_version, false) = false
+                            ORDER BY d.version_group_key, COALESCE(d.version_no, 1) DESC, d.id DESC
+                            """
+                        ),
+                        {"group_keys": group_keys},
+                    )
+                ).mappings().all()
+                for history_row in history_rows:
+                    group_key = str(history_row["version_group_key"] or "")
+                    history_by_group.setdefault(group_key, []).append(
+                        self._serialize_list_item_row(
+                            self._map_document_row(history_row),
+                            totalVersions=total_versions_by_group.get(group_key, 1),
+                            historyVersions=[],
+                        )
+                    )
+
         items = []
         for row in rows:
             mapped = self._map_document_row(row)
-            summary = self._build_summary_payload(
+            group_key = str(mapped.versionGroupKey or "")
-                mapped.totalScore,
+            total_versions = int(row.get("total_versions") or 1)
-                mapped.passedCount,
+            history_versions = history_by_group.get(group_key, []) if group_key else []
-                mapped.failedCount,
-                mapped.skippedCount,
-            )
             items.append(
-                {
+                self._serialize_list_item_row(
-                    "documentId": mapped.documentId,
+                    mapped,
-                    "fileId": mapped.fileId,
+                    totalVersions=total_versions,
-                    "fileName": mapped.fileName,
+                    historyVersions=history_versions,
-                    "fileExt": mapped.fileExt,
+                )
-                    "mimeType": mapped.mimeType,
-                    "fileSize": mapped.fileSize,
-                    "region": mapped.region,
-                    "processingStatus": mapped.processingStatus,
-                    "currentRunId": mapped.currentRunId,
-                    "latestRunId": mapped.currentRunId,
-                    "resultStatus": mapped.resultStatus,
-                    "score": float(mapped.totalScore) if mapped.totalScore is not None else None,
-                    "passedCount": mapped.passedCount or 0,
-                    "failedCount": mapped.failedCount or 0,
-                    "skippedCount": mapped.skippedCount or 0,
-                    "latestRun": {
-                        "runId": mapped.currentRunId,
-                        "summary": summary,
-                    } if mapped.currentRunId else None,
-                    "reports": {
-                        "hasHtmlReport": mapped.hasHtmlReport,
-                        "hasDocxReport": mapped.hasDocxReport,
-                    },
-                    "createdAt": self._iso(mapped.createdAt),
-                    "updatedAt": self._iso(mapped.updatedAt),
-                }
             )
 
         return {"items": items, "total": total, "page": page, "pageSize": pageSize}
@@ -1406,6 +1509,67 @@ class GovdocServiceImpl(IGovdocService):
             "skipped_count": int(skippedCount or 0),
         }
 
+    async def _find_govdoc_latest_version_candidate(
+        self,
+        session,
+        *,
+        typeId: int | None,
+        region: str,
+        normalizedName: str,
+        fileExt: str | None,
+    ) -> dict[str, Any] | None:
+        if typeId is not None:
+            candidate = await _find_latest_version_candidate(
+                session,
+                type_id=int(typeId),
+                root_group_id=None,
+                region=region,
+                normalized_name=normalizedName,
+                file_ext=fileExt,
+            )
+            if candidate:
+                return candidate
+
+        ext_clause = ""
+        params: dict[str, Any] = {
+            "region": region,
+            "normalized_name": normalizedName,
+        }
+        if fileExt:
+            ext_clause = " AND LOWER(COALESCE(f.file_ext, '')) = :file_ext"
+            params["file_ext"] = fileExt.lower()
+
+        row = (
+            await session.execute(
+                text(
+                    f"""
+                    SELECT
+                        d.id AS document_id,
+                        d.version_group_key,
+                        d.version_no,
+                        d.root_version_id,
+                        f.id AS file_id,
+                        f.sha256
+                    FROM leaudit_documents d
+                    JOIN leaudit_document_files f
+                      ON f.document_id = d.id
+                     AND f.is_active = true
+                     AND f.file_role = 'original'
+                     AND f.deleted_at IS NULL
+                    WHERE d.region = :region
+                      AND d.normalized_name = :normalized_name
+                      AND COALESCE(d.engine_type, 'leaudit') = 'govdoc'
+                      AND COALESCE(d.is_latest_version, true) = true
+                      AND d.deleted_at IS NULL{ext_clause}
+                    ORDER BY COALESCE(d.version_no, 1) DESC, d.id DESC
+                    LIMIT 1
+                    """
+                ),
+                params,
+            )
+        ).mappings().first()
+        return dict(row) if row else None
+
     def _group_artifacts_by_run(self, rows: list[Any]) -> dict[int, dict[str, Any]]:
         grouped: dict[int, dict[str, Any]] = {}
         artifactTypeMap = {
@@ -1445,12 +1609,76 @@ class GovdocServiceImpl(IGovdocService):
             )
         return grouped
 
+    def _build_list_summary_payload(self, row: _GovdocDocumentRow) -> dict[str, Any]:
+        summary = self._parse_json(row.resultSummaryJson) or {}
+        if not isinstance(summary, dict):
+            summary = {}
+        bySeverity = summary.get("by_severity")
+        byCategory = summary.get("by_category")
+        return {
+            "score": float(summary.get("score") or row.totalScore or 0),
+            "total_findings": int(summary.get("total_findings") or row.failedCount or 0),
+            "by_severity": bySeverity if isinstance(bySeverity, dict) else {},
+            "by_category": byCategory if isinstance(byCategory, dict) else {},
+            "passed_count": int(summary.get("passed_count") or row.passedCount or 0),
+            "failed_count": int(summary.get("failed_count") or row.failedCount or 0),
+            "skipped_count": int(summary.get("skipped_count") or row.skippedCount or 0),
+        }
+
+    def _serialize_list_item_row(
+        self,
+        row: _GovdocDocumentRow,
+        *,
+        totalVersions: int | None,
+        historyVersions: list[dict[str, Any]],
+    ) -> dict[str, Any]:
+        summary = self._build_list_summary_payload(row)
+        return {
+            "documentId": row.documentId,
+            "fileId": row.fileId,
+            "fileName": row.fileName,
+            "fileExt": row.fileExt,
+            "mimeType": row.mimeType,
+            "fileSize": row.fileSize,
+            "region": row.region,
+            "processingStatus": row.processingStatus,
+            "currentRunId": row.currentRunId,
+            "latestRunId": row.currentRunId,
+            "resultStatus": row.resultStatus,
+            "score": float(row.totalScore) if row.totalScore is not None else None,
+            "passedCount": row.passedCount or 0,
+            "failedCount": row.failedCount or 0,
+            "skippedCount": row.skippedCount or 0,
+            "versionGroupKey": row.versionGroupKey or "",
+            "versionNo": int(row.versionNo or 1),
+            "rootVersionId": int(row.rootVersionId or row.documentId),
+            "previousVersionId": int(row.previousVersionId) if row.previousVersionId is not None else None,
+            "totalVersions": int(totalVersions or max(1, len(historyVersions) + 1)),
+            "historyCount": len(historyVersions),
+            "historyVersions": historyVersions,
+            "latestRun": {
+                "runId": row.currentRunId,
+                "summary": summary,
+            } if row.currentRunId else None,
+            "reports": {
+                "hasHtmlReport": row.hasHtmlReport,
+                "hasDocxReport": row.hasDocxReport,
+            },
+            "createdAt": self._iso(row.createdAt),
+            "updatedAt": self._iso(row.updatedAt),
+        }
+
     def _map_document_row(self, row: Any) -> _GovdocDocumentRow:
         return _GovdocDocumentRow(
             documentId=int(row["document_id"]),
             region=str(row["region"] or "default"),
             processingStatus=str(row["processing_status"] or "waiting"),
             currentRunId=int(row["current_run_id"]) if row.get("current_run_id") is not None else None,
+            versionGroupKey=str(row["version_group_key"]) if row.get("version_group_key") else None,
+            versionNo=int(row.get("version_no") or 1),
+            rootVersionId=int(row["root_version_id"]) if row.get("root_version_id") is not None else None,
+            previousVersionId=int(row["previous_version_id"]) if row.get("previous_version_id") is not None else None,
+            isLatestVersion=bool(row.get("is_latest_version", True)),
             createdAt=row.get("created_at"),
             updatedAt=row.get("updated_at"),
             fileId=int(row["file_id"]),
@@ -1465,6 +1693,7 @@ class GovdocServiceImpl(IGovdocService):
             passedCount=int(row["passed_count"]) if row.get("passed_count") is not None else None,
             failedCount=int(row["failed_count"]) if row.get("failed_count") is not None else None,
             skippedCount=int(row["skipped_count"]) if row.get("skipped_count") is not None else None,
+            resultSummaryJson=row.get("result_summary_json"),
             hasHtmlReport=bool(row.get("has_html_report")),
             hasDocxReport=bool(row.get("has_docx_report")),
         )