feat: restore rag dataset management and linkage

2026-05-11 17:21:33 +08:00
parent da2bb8310d
commit dcc0f3c30d
6 changed files with 2208 additions and 46 deletions
@@ -2,7 +2,9 @@ from __future__ import annotations

 from typing import Any

-from fastapi import Depends, Query
+import json
+
+from fastapi import Depends, File, Form, Query, UploadFile
 from fastapi.responses import JSONResponse, StreamingResponse

 from fastapi_common.fastapi_common_security.security import verify_access_token
@@ -14,6 +16,7 @@ from fastapi_modules.fastapi_leaudit.domian.Dto.ragChatDto import (
    RagChatSendMessageDTO,
    RagMessageFeedbackDTO,
 )
+from fastapi_modules.fastapi_leaudit.domian.Dto.ragDatasetDto import RagDatasetUpdateDTO
 from fastapi_modules.fastapi_leaudit.domian.vo.ragChatVo import (
    RagAppParametersVO,
    RagChatAppListVO,
@@ -23,7 +26,16 @@ from fastapi_modules.fastapi_leaudit.domian.vo.ragChatVo import (
    RagMessagePageVO,
    RagOperationResultVO,
 )
-from fastapi_modules.fastapi_leaudit.domian.vo.ragDatasetVo import RagDatasetPageVO
+from fastapi_modules.fastapi_leaudit.domian.vo.ragDatasetVo import (
+    RagDatasetDetailVO,
+    RagDatasetDocumentItemVO,
+    RagDatasetDocumentPageVO,
+    RagDatasetPageVO,
+    RagDatasetRetrieveResponseVO,
+    RagDatasetSegmentItemVO,
+    RagDatasetSegmentPageVO,
+    RagDatasetUploadDocumentVO,
+)
 from fastapi_modules.fastapi_leaudit.services.impl.permissionServiceImpl import PermissionServiceImpl
 from fastapi_modules.fastapi_leaudit.services.impl.ragChatServiceImpl import RagChatServiceImpl
 from fastapi_modules.fastapi_leaudit.services.impl.ragDatasetServiceImpl import RagDatasetServiceImpl
@@ -82,6 +94,332 @@ class RagChatController(BaseController):
            )
            return Result.success(data=data)

+        @self.router.get("/datasets/admin", response_model=Result[RagDatasetPageVO])
+        async def GetAdminDatasets(
+            area: str | None = Query(None),
+            onlyEnabled: bool | None = Query(None),
+            page: int = Query(1, ge=1),
+            pageSize: int = Query(20, ge=1, le=200),
+            payload: dict[str, Any] = Depends(verify_access_token),
+        ):
+            if not await self._check_permission(int(payload["user_id"]), [self._PERMISSIONS["dataset_read"]]):
+                return JSONResponse(status_code=403, content={"code": 403, "msg": "当前用户没有管理知识库权限", "data": None})
+            data = await self.RagDatasetService.GetAdminDatasets(
+                CurrentUserId=int(payload["user_id"]),
+                UserArea=payload.get("area"),
+                UserRole=payload.get("user_role"),
+                Area=area,
+                OnlyEnabled=onlyEnabled,
+                Page=page,
+                PageSize=pageSize,
+            )
+            return Result.success(data=data)
+
+        @self.router.post("/datasets/admin", response_model=Result[RagDatasetDetailVO])
+        async def CreateAdminDataset(Body: dict[str, Any], payload: dict[str, Any] = Depends(verify_access_token)):
+            if not await self._check_permission(int(payload["user_id"]), [self._PERMISSIONS["dataset_read"]]):
+                return JSONResponse(status_code=403, content={"code": 403, "msg": "当前用户没有创建知识库权限", "data": None})
+            data = await self.RagDatasetService.CreateAdminDataset(
+                CurrentUserId=int(payload["user_id"]),
+                UserArea=payload.get("area"),
+                UserRole=payload.get("user_role"),
+                Body=Body,
+            )
+            return Result.success(data=data)
+
+        @self.router.put("/datasets/admin/{DatasetId}", response_model=Result[RagDatasetDetailVO | None])
+        async def UpdateAdminDataset(DatasetId: int, Body: dict[str, Any], payload: dict[str, Any] = Depends(verify_access_token)):
+            if not await self._check_permission(int(payload["user_id"]), [self._PERMISSIONS["dataset_read"]]):
+                return JSONResponse(status_code=403, content={"code": 403, "msg": "当前用户没有更新知识库权限", "data": None})
+            data = await self.RagDatasetService.UpdateAdminDataset(
+                CurrentUserId=int(payload["user_id"]),
+                UserArea=payload.get("area"),
+                UserRole=payload.get("user_role"),
+                DatasetId=DatasetId,
+                Body=Body,
+            )
+            return Result.success(data=data)
+
+        @self.router.delete("/datasets/admin/{DatasetId}", response_model=Result[RagOperationResultVO])
+        async def DeleteAdminDataset(DatasetId: int, payload: dict[str, Any] = Depends(verify_access_token)):
+            if not await self._check_permission(int(payload["user_id"]), [self._PERMISSIONS["dataset_read"]]):
+                return JSONResponse(status_code=403, content={"code": 403, "msg": "当前用户没有删除知识库权限", "data": None})
+            data = await self.RagDatasetService.DeleteAdminDataset(
+                CurrentUserId=int(payload["user_id"]),
+                UserArea=payload.get("area"),
+                UserRole=payload.get("user_role"),
+                DatasetId=DatasetId,
+            )
+            return Result.success(data=data)
+
+        @self.router.get("/datasets/{DatasetId}", response_model=Result[RagDatasetDetailVO | None])
+        async def GetDatasetDetail(DatasetId: int, payload: dict[str, Any] = Depends(verify_access_token)):
+            if not await self._check_permission(int(payload["user_id"]), [self._PERMISSIONS["dataset_read"]]):
+                return JSONResponse(status_code=403, content={"code": 403, "msg": "当前用户没有查看知识库权限", "data": None})
+            data = await self.RagDatasetService.GetDatasetDetail(
+                CurrentUserId=int(payload["user_id"]),
+                UserArea=payload.get("area"),
+                UserRole=payload.get("user_role"),
+                DatasetId=DatasetId,
+            )
+            return Result.success(data=data)
+
+        @self.router.patch("/datasets/{DatasetId}", response_model=Result[RagDatasetDetailVO | None])
+        async def UpdateDataset(DatasetId: int, Body: RagDatasetUpdateDTO, payload: dict[str, Any] = Depends(verify_access_token)):
+            if not await self._check_permission(int(payload["user_id"]), [self._PERMISSIONS["dataset_read"]]):
+                return JSONResponse(status_code=403, content={"code": 403, "msg": "当前用户没有修改知识库权限", "data": None})
+            data = await self.RagDatasetService.UpdateDataset(
+                CurrentUserId=int(payload["user_id"]),
+                UserArea=payload.get("area"),
+                UserRole=payload.get("user_role"),
+                DatasetId=DatasetId,
+                Body=Body,
+            )
+            return Result.success(data=data)
+
+        @self.router.get("/datasets/{DatasetId}/documents", response_model=Result[RagDatasetDocumentPageVO])
+        async def GetDatasetDocuments(
+            DatasetId: int,
+            page: int = Query(1, ge=1),
+            limit: int = Query(20, ge=1, le=100),
+            keyword: str | None = Query(None),
+            payload: dict[str, Any] = Depends(verify_access_token),
+        ):
+            if not await self._check_permission(int(payload["user_id"]), [self._PERMISSIONS["dataset_read"]]):
+                return JSONResponse(status_code=403, content={"code": 403, "msg": "当前用户没有查看知识库文档权限", "data": None})
+            data = await self.RagDatasetService.GetDatasetDocuments(
+                CurrentUserId=int(payload["user_id"]),
+                UserArea=payload.get("area"),
+                UserRole=payload.get("user_role"),
+                DatasetId=DatasetId,
+                Page=page,
+                Limit=limit,
+                Keyword=keyword,
+            )
+            return Result.success(data=data)
+
+        @self.router.get("/datasets/{DatasetId}/documents/{DocumentId}", response_model=Result[RagDatasetDocumentItemVO | None])
+        async def GetDatasetDocumentDetail(
+            DatasetId: int,
+            DocumentId: int,
+            payload: dict[str, Any] = Depends(verify_access_token),
+        ):
+            if not await self._check_permission(int(payload["user_id"]), [self._PERMISSIONS["dataset_read"]]):
+                return JSONResponse(status_code=403, content={"code": 403, "msg": "当前用户没有查看知识库文档权限", "data": None})
+            data = await self.RagDatasetService.GetDatasetDocumentDetail(
+                CurrentUserId=int(payload["user_id"]),
+                UserArea=payload.get("area"),
+                UserRole=payload.get("user_role"),
+                DatasetId=DatasetId,
+                DocumentId=DocumentId,
+            )
+            return Result.success(data=data)
+
+        @self.router.post("/datasets/{DatasetId}/documents", response_model=Result[RagDatasetUploadDocumentVO])
+        async def UploadDatasetDocument(
+            DatasetId: int,
+            file: UploadFile = File(...),
+            data: str | None = Form(None),
+            payload: dict[str, Any] = Depends(verify_access_token),
+        ):
+            if not await self._check_permission(int(payload["user_id"]), [self._PERMISSIONS["dataset_read"]]):
+                return JSONResponse(status_code=403, content={"code": 403, "msg": "当前用户没有上传知识库文档权限", "data": None})
+            process_config = json.loads(data) if data else None
+            file_bytes = await file.read()
+            result = await self.RagDatasetService.UploadDatasetDocument(
+                CurrentUserId=int(payload["user_id"]),
+                UserArea=payload.get("area"),
+                UserRole=payload.get("user_role"),
+                DatasetId=DatasetId,
+                FileName=file.filename or "document",
+                ContentType=file.content_type,
+                Content=file_bytes,
+                ProcessConfig=process_config,
+            )
+            return Result.success(data=result)
+
+        @self.router.post("/datasets/{DatasetId}/documents/{DocumentId}/update-by-file", response_model=Result[RagDatasetUploadDocumentVO])
+        async def UpdateDatasetDocumentByFile(
+            DatasetId: int,
+            DocumentId: int,
+            file: UploadFile = File(...),
+            data: str | None = Form(None),
+            payload: dict[str, Any] = Depends(verify_access_token),
+        ):
+            if not await self._check_permission(int(payload["user_id"]), [self._PERMISSIONS["dataset_read"]]):
+                return JSONResponse(status_code=403, content={"code": 403, "msg": "当前用户没有重处理知识库文档权限", "data": None})
+            process_config = json.loads(data) if data else None
+            file_bytes = await file.read()
+            result = await self.RagDatasetService.UpdateDatasetDocumentByFile(
+                CurrentUserId=int(payload["user_id"]),
+                UserArea=payload.get("area"),
+                UserRole=payload.get("user_role"),
+                DatasetId=DatasetId,
+                DocumentId=DocumentId,
+                FileName=file.filename or "document",
+                ContentType=file.content_type,
+                Content=file_bytes,
+                ProcessConfig=process_config,
+            )
+            return Result.success(data=result)
+
+        @self.router.get("/datasets/{DatasetId}/documents/{DocumentId}/indexing-status")
+        async def GetDatasetDocumentIndexingStatus(
+            DatasetId: int,
+            DocumentId: int,
+            payload: dict[str, Any] = Depends(verify_access_token),
+        ):
+            if not await self._check_permission(int(payload["user_id"]), [self._PERMISSIONS["dataset_read"]]):
+                return JSONResponse(status_code=403, content={"code": 403, "msg": "当前用户没有查看知识库处理进度权限", "data": None})
+            result = await self.RagDatasetService.GetDatasetDocumentIndexingStatus(
+                CurrentUserId=int(payload["user_id"]),
+                UserArea=payload.get("area"),
+                UserRole=payload.get("user_role"),
+                DatasetId=DatasetId,
+                DocumentId=DocumentId,
+            )
+            return Result.success(data=result["data"])
+
+        @self.router.patch("/datasets/{DatasetId}/documents/status/{Action}", response_model=Result[RagOperationResultVO])
+        async def BatchUpdateDatasetDocumentStatus(
+            DatasetId: int,
+            Action: str,
+            Body: dict[str, Any],
+            payload: dict[str, Any] = Depends(verify_access_token),
+        ):
+            if not await self._check_permission(int(payload["user_id"]), [self._PERMISSIONS["dataset_read"]]):
+                return JSONResponse(status_code=403, content={"code": 403, "msg": "当前用户没有修改知识库文档状态权限", "data": None})
+            enabled = Action == "enable"
+            if Action not in {"enable", "disable"}:
+                return JSONResponse(status_code=400, content={"code": 400, "msg": "当前仅支持启用和禁用", "data": None})
+            document_ids = Body.get("document_ids") or []
+            result = await self.RagDatasetService.BatchUpdateDatasetDocumentStatus(
+                CurrentUserId=int(payload["user_id"]),
+                UserArea=payload.get("area"),
+                UserRole=payload.get("user_role"),
+                DatasetId=DatasetId,
+                DocumentIds=[int(item) for item in document_ids],
+                Enabled=enabled,
+            )
+            return Result.success(data=result)
+
+        @self.router.get("/datasets/{DatasetId}/documents/{DocumentId}/segments", response_model=Result[RagDatasetSegmentPageVO])
+        async def GetDatasetDocumentSegments(
+            DatasetId: int,
+            DocumentId: int,
+            page: int = Query(1, ge=1),
+            limit: int = Query(20, ge=1, le=200),
+            keyword: str | None = Query(None),
+            payload: dict[str, Any] = Depends(verify_access_token),
+        ):
+            if not await self._check_permission(int(payload["user_id"]), [self._PERMISSIONS["dataset_read"]]):
+                return JSONResponse(status_code=403, content={"code": 403, "msg": "当前用户没有查看知识库分段权限", "data": None})
+            result = await self.RagDatasetService.GetDatasetDocumentSegments(
+                CurrentUserId=int(payload["user_id"]),
+                UserArea=payload.get("area"),
+                UserRole=payload.get("user_role"),
+                DatasetId=DatasetId,
+                DocumentId=DocumentId,
+                Page=page,
+                Limit=limit,
+                Keyword=keyword,
+            )
+            return Result.success(data=result)
+
+        @self.router.delete("/datasets/{DatasetId}/documents/{DocumentId}", response_model=Result[RagOperationResultVO])
+        async def DeleteDatasetDocument(
+            DatasetId: int,
+            DocumentId: int,
+            payload: dict[str, Any] = Depends(verify_access_token),
+        ):
+            if not await self._check_permission(int(payload["user_id"]), [self._PERMISSIONS["dataset_read"]]):
+                return JSONResponse(status_code=403, content={"code": 403, "msg": "当前用户没有删除知识库文档权限", "data": None})
+            result = await self.RagDatasetService.DeleteDatasetDocument(
+                CurrentUserId=int(payload["user_id"]),
+                UserArea=payload.get("area"),
+                UserRole=payload.get("user_role"),
+                DatasetId=DatasetId,
+                DocumentId=DocumentId,
+            )
+            return Result.success(data=result)
+
+        @self.router.post("/datasets/{DatasetId}/retrieve", response_model=Result[RagDatasetRetrieveResponseVO])
+        async def RetrieveDataset(
+            DatasetId: int,
+            Body: dict[str, Any],
+            payload: dict[str, Any] = Depends(verify_access_token),
+        ):
+            if not await self._check_permission(int(payload["user_id"]), [self._PERMISSIONS["dataset_read"]]):
+                return JSONResponse(status_code=403, content={"code": 403, "msg": "当前用户没有检索知识库权限", "data": None})
+            result = await self.RagDatasetService.RetrieveDataset(
+                CurrentUserId=int(payload["user_id"]),
+                UserArea=payload.get("area"),
+                UserRole=payload.get("user_role"),
+                DatasetId=DatasetId,
+                Query=str(Body.get("query") or ""),
+                RetrievalModel=Body.get("retrieval_model") if isinstance(Body.get("retrieval_model"), dict) else None,
+            )
+            return Result.success(data=result)
+
+        @self.router.get("/datasets/{DatasetId}/documents/{DocumentId}/segments/{SegmentId}", response_model=Result[RagDatasetSegmentItemVO | None])
+        async def GetDatasetDocumentSegmentDetail(
+            DatasetId: int,
+            DocumentId: int,
+            SegmentId: str,
+            payload: dict[str, Any] = Depends(verify_access_token),
+        ):
+            if not await self._check_permission(int(payload["user_id"]), [self._PERMISSIONS["dataset_read"]]):
+                return JSONResponse(status_code=403, content={"code": 403, "msg": "当前用户没有查看知识库分段权限", "data": None})
+            result = await self.RagDatasetService.GetDatasetDocumentSegmentDetail(
+                CurrentUserId=int(payload["user_id"]),
+                UserArea=payload.get("area"),
+                UserRole=payload.get("user_role"),
+                DatasetId=DatasetId,
+                DocumentId=DocumentId,
+                SegmentId=SegmentId,
+            )
+            return Result.success(data=result)
+
+        @self.router.post("/datasets/{DatasetId}/documents/{DocumentId}/segments/{SegmentId}", response_model=Result[RagDatasetSegmentItemVO | None])
+        async def UpdateDatasetDocumentSegment(
+            DatasetId: int,
+            DocumentId: int,
+            SegmentId: str,
+            Body: dict[str, Any],
+            payload: dict[str, Any] = Depends(verify_access_token),
+        ):
+            if not await self._check_permission(int(payload["user_id"]), [self._PERMISSIONS["dataset_read"]]):
+                return JSONResponse(status_code=403, content={"code": 403, "msg": "当前用户没有修改知识库分段权限", "data": None})
+            result = await self.RagDatasetService.UpdateDatasetDocumentSegment(
+                CurrentUserId=int(payload["user_id"]),
+                UserArea=payload.get("area"),
+                UserRole=payload.get("user_role"),
+                DatasetId=DatasetId,
+                DocumentId=DocumentId,
+                SegmentId=SegmentId,
+                Body=Body,
+            )
+            return Result.success(data=result)
+
+        @self.router.delete("/datasets/{DatasetId}/documents/{DocumentId}/segments/{SegmentId}", response_model=Result[RagOperationResultVO])
+        async def DeleteDatasetDocumentSegment(
+            DatasetId: int,
+            DocumentId: int,
+            SegmentId: str,
+            payload: dict[str, Any] = Depends(verify_access_token),
+        ):
+            if not await self._check_permission(int(payload["user_id"]), [self._PERMISSIONS["dataset_read"]]):
+                return JSONResponse(status_code=403, content={"code": 403, "msg": "当前用户没有删除知识库分段权限", "data": None})
+            result = await self.RagDatasetService.DeleteDatasetDocumentSegment(
+                CurrentUserId=int(payload["user_id"]),
+                UserArea=payload.get("area"),
+                UserRole=payload.get("user_role"),
+                DatasetId=DatasetId,
+                DocumentId=DocumentId,
+                SegmentId=SegmentId,
+            )
+            return Result.success(data=result)
+
        @self.router.get("/chat/parameters", response_model=Result[RagAppParametersVO])
        async def GetAppParameters(
            appId: int | None = Query(None, description="聊天应用ID"),