From 9c86bf59e5c50132580d3197f9d172cdfeeb0d82 Mon Sep 17 00:00:00 2001 From: wren <“porlong@qq.com”> Date: Fri, 8 May 2026 10:58:24 +0800 Subject: [PATCH] feat: add rag backend and review access fixes --- docs/RAG后端上线清单.md | 310 +++++++ docs/团队Git协作完整规范-2026-05-07.md | 852 ++++++++++++++++++ docs/接口/RAG聊天接口.md | 493 ++++++++++ docs/接口/README.md | 1 + .../fastapi_common_security/jwtService.py | 4 +- .../controllers/ragChatController.py | 173 ++++ .../fastapi_leaudit/domian/Dto/ragChatDto.py | 15 + .../fastapi_leaudit/domian/vo/ragChatVo.py | 59 ++ .../fastapi_leaudit/domian/vo/ragDatasetVo.py | 18 + .../fastapi_leaudit/models/__init__.py | 10 + .../models/leauditRagChatApp.py | 27 + .../models/leauditRagConversation.py | 17 + .../models/leauditRagDataset.py | 30 + .../models/leauditRagDocument.py | 24 + .../models/leauditRagMessage.py | 20 + .../fastapi_leaudit/rag_engine/__init__.py | 1 + .../rag_engine/chroma_client.py | 40 + .../fastapi_leaudit/rag_engine/config.py | 60 ++ .../fastapi_leaudit/rag_engine/generator.py | 144 +++ .../rag_engine/question_chains.py | 39 + .../fastapi_leaudit/services/__init__.py | 4 + .../services/impl/documentServiceImpl.py | 77 +- .../services/impl/ragChatServiceImpl.py | 589 ++++++++++++ .../services/impl/ragDatasetServiceImpl.py | 52 ++ .../services/impl/rbacAdminServiceImpl.py | 7 + .../services/ragChatService.py | 62 ++ .../services/ragDatasetService.py | 10 + pyproject.toml | 1 + scripts/merge_document_version_groups.py | 277 ++++++ scripts/preview_document_version_merge.py | 236 +++++ scripts/schema_add_rag_chat.sql | 187 ++++ scripts/user_rbac_seed.sql | 61 +- 32 files changed, 3877 insertions(+), 23 deletions(-) create mode 100644 docs/RAG后端上线清单.md create mode 100644 docs/团队Git协作完整规范-2026-05-07.md create mode 100644 docs/接口/RAG聊天接口.md create mode 100644 fastapi_modules/fastapi_leaudit/controllers/ragChatController.py create mode 100644 fastapi_modules/fastapi_leaudit/domian/Dto/ragChatDto.py create mode 100644 fastapi_modules/fastapi_leaudit/domian/vo/ragChatVo.py create mode 100644 fastapi_modules/fastapi_leaudit/domian/vo/ragDatasetVo.py create mode 100644 fastapi_modules/fastapi_leaudit/models/leauditRagChatApp.py create mode 100644 fastapi_modules/fastapi_leaudit/models/leauditRagConversation.py create mode 100644 fastapi_modules/fastapi_leaudit/models/leauditRagDataset.py create mode 100644 fastapi_modules/fastapi_leaudit/models/leauditRagDocument.py create mode 100644 fastapi_modules/fastapi_leaudit/models/leauditRagMessage.py create mode 100644 fastapi_modules/fastapi_leaudit/rag_engine/__init__.py create mode 100644 fastapi_modules/fastapi_leaudit/rag_engine/chroma_client.py create mode 100644 fastapi_modules/fastapi_leaudit/rag_engine/config.py create mode 100644 fastapi_modules/fastapi_leaudit/rag_engine/generator.py create mode 100644 fastapi_modules/fastapi_leaudit/rag_engine/question_chains.py create mode 100644 fastapi_modules/fastapi_leaudit/services/impl/ragChatServiceImpl.py create mode 100644 fastapi_modules/fastapi_leaudit/services/impl/ragDatasetServiceImpl.py create mode 100644 fastapi_modules/fastapi_leaudit/services/ragChatService.py create mode 100644 fastapi_modules/fastapi_leaudit/services/ragDatasetService.py create mode 100644 scripts/merge_document_version_groups.py create mode 100644 scripts/preview_document_version_merge.py create mode 100644 scripts/schema_add_rag_chat.sql diff --git a/docs/RAG后端上线清单.md b/docs/RAG后端上线清单.md new file mode 100644 index 0000000..758f80d --- /dev/null +++ b/docs/RAG后端上线清单.md @@ -0,0 +1,310 @@ +# RAG 后端上线清单 + +适用范围： +- 仓库：`leaudit-platform` +- 本次上线内容：自有 RAG 聊天后端、`/api/v3/rag/*` 接口、`rag_*` 表结构、`rag:*` 权限 + +## 1. 上线前确认 + +- 代码已包含以下变更： + - `fastapi_modules/fastapi_leaudit/controllers/ragChatController.py` + - `fastapi_modules/fastapi_leaudit/services/impl/ragChatServiceImpl.py` + - `fastapi_modules/fastapi_leaudit/services/impl/ragDatasetServiceImpl.py` + - `scripts/schema_add_rag_chat.sql` + - `scripts/user_rbac_seed.sql` + - `pyproject.toml` 已加入 `chromadb` +- 发布分支已经合并到目标部署分支。 +- 数据库备份已完成，至少备份： + - `permissions` + - `role_permissions` + - `role_route` + - `sys_routes` + - 全部 `rag_*` 表 + +## 2. 配置优先级 + +后端配置加载顺序见 `fastapi_admin/config/_loader.py`： + +1. `app.toml` +2. `app.{APP_ENV}.toml` +3. `app.ai.toml` +4. 已存在的环境变量 + +结论： +- 生产环境如果通过环境变量注入，优先级最高。 +- 如果继续使用 TOML，建议把生产密钥放到 `app.ai.toml` 或外部环境变量，不要直接把敏感值写死在默认 `app.toml`。 + +## 3. 必备环境变量 / 配置项 + +### 3.1 基础服务配置 + +来自 `fastapi_admin/config/_settings.py`： + +- `APP_HOST` +- `APP_PORT` +- `JWT_SECRET_KEY` +- `DB_HOST` +- `DB_PORT` +- `DB_NAME` +- `DB_USER` +- `DB_PASSWORD` +- `REDIS_HOST` +- `REDIS_PORT` +- `REDIS_DB` +- `REDIS_PASSWORD` +- `OSS_ENDPOINT` +- `OSS_BASE_URL` +- `OSS_ACCESS_KEY` +- `OSS_SECRET_KEY` +- `OSS_BUCKET` + +### 3.2 RAG / LLM 配置 + +来自 `fastapi_modules/fastapi_leaudit/rag_engine/config.py`： + +- `LLM_BASE_URL` +- `LLM_MODEL` +- `LLM_API_KEY` +- `RAG_LLM_TEMPERATURE` +- `RAG_LLM_MAX_TOKENS` +- `RAG_LLM_TIMEOUT` + +### 3.3 Chroma 配置 + +二选一： + +- 远程 Chroma： + - `RAG_CHROMA_HOST` + - `RAG_CHROMA_PORT` + - `RAG_CHROMA_TOKEN`（如有鉴权） + - `RAG_CHROMA_AUTH_HEADER`（默认 `X-Chroma-Token`） +- 本地持久化 Chroma： + - `RAG_CHROMA_PERSIST_DIR` + +说明： +- 如果不配置 `RAG_CHROMA_HOST`，代码会走本地 `PersistentClient`。 +- 如果 Chroma 依赖缺失或不可用，服务能启动，但检索会退化成“无上下文回答”。 + +### 3.4 可选检索参数 + +- `RAG_VECTOR_TOP_K` +- `RAG_RERANK_TOP_K` +- `RAG_BM25_TOP_K` +- `RAG_RRF_K` +- `RAG_QUERY_REWRITING` +- `RAG_HYBRID_SEARCH` +- `RAG_RERANKING` +- `RAG_EMBED_URL` +- `RAG_EMBED_KEY` +- `RAG_EMBED_MODEL` +- `RAG_EMBED_DIM` +- `RAG_EMBED_BATCH_SIZE` +- `RAG_RERANKER_URL` +- `RAG_RERANKER_KEY` +- `RAG_RERANKER_MODEL` + +说明： +- 当前聊天主流程已经能工作；即使没有完整 embedding/reranker 配置，只要现有 Chroma 集合可用，也可以先上线基础聊天。 +- 更细的召回参数也可以放在 `rag_dataset.retrieval_model` 里按知识库配置。 + +## 4. 依赖安装 + +### 4.1 Python 依赖 + +本次新增依赖： + +- `chromadb>=0.5.23` + +如果线上使用虚拟环境： + +```bash +source .venv/bin/activate +pip install -e . +``` + +如果只补增量依赖： + +```bash +source .venv/bin/activate +pip install chromadb +``` + +建议最终仍执行一次完整依赖同步，避免环境漂移。 + +## 5. 数据库执行顺序 + +### 5.1 先执行结构脚本 + +```bash +psql -v ON_ERROR_STOP=1 \ + -h -p -U -d \ + -f scripts/schema_add_rag_chat.sql +``` + +作用： +- 创建 `rag_dataset / rag_document / rag_chat_app / rag_conversation / rag_message` +- 对已存在表补字段、补索引、补 `updated_at` 触发器 + +### 5.2 再执行权限种子 + +```bash +psql -v ON_ERROR_STOP=1 \ + -h -p -U -d \ + -f scripts/user_rbac_seed.sql +``` + +作用： +- 补 `rag:*` 权限 +- 补角色授权 +- 同步需要的菜单/路由元数据 + +### 5.3 当前已验证结果 + +本地已在目标库执行并验证： + +- 已存在 5 张 `rag_*` 表 +- 已存在 7 条 `rag:*` 权限 +- `super_admin / provincial_admin / admin / common` 均已获得这 7 条权限 + +## 6. 重启顺序 + +推荐顺序如下。 + +### 步骤 1：停 Celery worker + +如果当前有 worker 在跑，先停掉，避免旧代码消费任务： + +```bash +pkill -f "celery.*fastapi_admin.celery_app:celery_app" || true +``` + +如果你们线上是用进程管理器托管 worker，则用对应管理命令停止。 + +### 步骤 2：更新代码并安装依赖 + +```bash +git pull +source .venv/bin/activate +pip install -e . +``` + +### 步骤 3：执行数据库脚本 + +按“第 5 节”的先后顺序执行： + +1. `scripts/schema_add_rag_chat.sql` +2. `scripts/user_rbac_seed.sql` + +### 步骤 4：重启 FastAPI 主服务 + +如果用 `uvicorn` 直接启动，参考 `run.py`： + +```bash +source .venv/bin/activate +python run.py +``` + +如果线上不是 `reload=True` 的开发模式，建议使用你们现有的生产启动命令重启主进程，不要直接照抄开发命令。 + +### 步骤 5：重启 Celery worker + +仓库已提供脚本 `scripts/start_worker.sh`： + +```bash +./scripts/start_worker.sh +``` + +它会自动读取： +- `LEAUDIT_WORKER_CONCURRENCY` +- `LEAUDIT_WORKER_QUEUE_URGENT` +- `LEAUDIT_WORKER_QUEUE_NORMAL` + +### 步骤 6：如有反向代理，再做一次健康检查 + +- Nginx / 网关 upstream 是否恢复 +- 80/443 -> FastAPI upstream 是否正常 + +## 7. 上线后验证 + +### 7.1 代码级快速检查 + +```bash +python -m compileall fastapi_modules/fastapi_leaudit +``` + +### 7.2 路由检查 + +已知应存在 10 条 `/v3/rag` 路由： + +- `GET /api/v3/rag/apps` +- `GET /api/v3/rag/apps/default` +- `GET /api/v3/rag/datasets/my` +- `GET /api/v3/rag/chat/parameters` +- `POST /api/v3/rag/chat/messages` +- `GET /api/v3/rag/chat/conversations` +- `GET /api/v3/rag/chat/conversations/{ConversationId}/messages` +- `PATCH /api/v3/rag/chat/conversations/{ConversationId}` +- `DELETE /api/v3/rag/chat/conversations/{ConversationId}` +- `POST /api/v3/rag/chat/messages/{MessageId}/feedback` + +### 7.3 数据库检查 + +```sql +SELECT tablename +FROM pg_tables +WHERE schemaname = 'public' AND tablename LIKE 'rag_%' +ORDER BY tablename; +``` + +```sql +SELECT count(*) +FROM permissions +WHERE permission_key LIKE 'rag:%'; +``` + +```sql +SELECT r.role_key, count(*) +FROM role_permissions rp +JOIN roles r ON r.id = rp.role_id +JOIN permissions p ON p.id = rp.permission_id +WHERE p.permission_key LIKE 'rag:%' +GROUP BY r.role_key +ORDER BY r.role_key; +``` + +### 7.4 接口联通检查 + +至少验证这 4 个接口： + +1. `GET /api/v3/rag/apps` +2. `GET /api/v3/rag/apps/default` +3. `GET /api/v3/rag/chat/parameters` +4. `POST /api/v3/rag/chat/messages` + +关注点： +- 401：JWT 或登录态异常 +- 403：`rag:*` 权限未生效 +- 404：控制器未注册 / 网关路径未转发 +- 500：LLM、Chroma、数据库连接或 JSON 字段异常 + +## 8. 常见风险 + +- `chromadb` 未安装：服务启动不一定报错，但首次检索时会退化 +- Chroma 集合为空：可以聊天，但没有知识库上下文 +- `LLM_BASE_URL / LLM_API_KEY / LLM_MODEL` 配错：流式对话会直接失败 +- 只执行权限脚本、不执行结构脚本：接口可能启动，但落库时报错 +- 只重启 API、不重启 worker：后台异步任务仍可能跑旧代码 + +## 9. 建议的最短上线动作 + +按最快可落地顺序： + +1. 备份数据库 +2. `git pull` +3. `source .venv/bin/activate && pip install -e .` +4. 执行 `scripts/schema_add_rag_chat.sql` +5. 执行 `scripts/user_rbac_seed.sql` +6. 重启 FastAPI 主服务 +7. 重启 `scripts/start_worker.sh` +8. 验证 `/api/v3/rag/apps` 和 `/api/v3/rag/chat/messages` + diff --git a/docs/团队Git协作完整规范-2026-05-07.md b/docs/团队Git协作完整规范-2026-05-07.md new file mode 100644 index 0000000..3b2fac8 --- /dev/null +++ b/docs/团队Git协作完整规范-2026-05-07.md @@ -0,0 +1,852 @@ +# 团队 Git 协作完整规范 + +> 适用对象：个人项目、小项目、大项目 +> 版本：v1.0 · 2026-05-07 + +--- + +## 一、核心原则 + +不论团队规模大小，所有 Git 协作都遵守以下 5 条铁律： + +1. **主干（main）永远可发布** +2. **凡是会影响他人的操作，事前通知、事中可见、事后可恢复** +3. **分支属于"任务"，不属于"人"**（详见第六章） +4. **推送前先同步**：`git pull --rebase` 后再 `git push` +5. **重要分支禁止直接 push，必须走 PR** + +--- + +## 二、如何选择工作流 + +### 决策树 + +``` +你一个人做这个项目吗？ +├─ 是 → 【方案 A】个人极简流 +└─ 否 → 团队 ≥ 6 人 或 同时维护多版本？ + ├─ 是 → 【方案 C】Git Flow Lite（带 develop） + └─ 否 → 【方案 B】GitHub Flow（90% 小项目适用） +``` + +### 三种方案对比 + +| 维度 | A·个人 | B·小项目 | C·大项目 | +|------|--------|---------|---------| +| 团队规模 | 1 人 | 2-5 人 | 6+ 人 | +| 主分支 | main | main | main + develop | +| 是否需 develop | ❌ | ❌ | ✅ | +| 是否需 release 分支 | ❌ | ❌ | ✅（可选） | +| 是否需 hotfix 分支 | ❌ | ✅ | ✅ | +| 部署策略 | 手动 / tag | 持续部署 | 阶段部署 | +| 发布周期 | 不固定 | 随时 | 双周 / 月度 | +| PR Review | 自看 | ≥1 人 | ≥2 人 | +| 复杂度 | ★ | ★★ | ★★★★ | + +### 升级信号 + +**A → B**：第二个人加入，立刻配 main 保护 + PR 流程。 + +**B → C**：满足以下任一即升级 +- 同时维护多个版本（v1 还在用，v2 在开发） +- 有专门 QA 阶段，main 不能合并即上线 +- 发布周期 ≥ 2 周 +- 团队 ≥ 6 人 + +> ⚠️ **宁可滞后升级，也不要预防性复杂化**。多数团队过度设计，把简单事搞复杂。 + +--- + +## 三、方案 A：个人项目工作流 + +### 分支模型 + +``` +main (主线 + 发布锚点) +└─ feature/* (按需，仅用于实验性大改动) +``` + +### 日常工作流 + +**小改动 → 直接上 main** + +```bash +git add . +git commit -m "fix: 修复登录按钮样式" +git push +``` + +**大改动 / 不确定的实验 → 开分支保命** + +```bash +git checkout -b feature/重构数据层 +# ...开发... + +# 没问题 → 合并 +git checkout main +git merge --no-ff feature/重构数据层 +git push +git branch -d feature/重构数据层 + +# 实验失败 → 直接丢弃 +git checkout main +git branch -D feature/重构数据层 +``` + +### 发布流程 + +```bash +git tag -a v1.2.0 -m "release: v1.2.0 - 评查详情页上线" +git push --tags +``` + +**版本号遵循 [SemVer](https://semver.org/)**： +- `v主版本.次版本.补丁` +- 主版本：不兼容改动 +- 次版本：新功能（向下兼容） +- 补丁：bug 修复 + +### 个人项目最低纪律 + +| 习惯 | 价值 | +|------|------| +| 每天 push 到远程 | 硬盘坏了不丢工 | +| commit 信息写人话 | 半年后能看懂 | +| 重大改动前打 tag | 有回滚锚点 | +| README 记录关键决策 | 时间久了不抓瞎 | +| `.gitignore` 配齐 | 不把密钥推上去 | + +### 不要做的事 + +- ❌ `git commit -m "update"` / `"修改"` / `"123"` +- ❌ 一周不 push +- ❌ 在 main 上做不确定的实验 +- ❌ 把 `.env` 推到远程 + +--- + +## 四、方案 B：小项目工作流（GitHub Flow） + +**适用：2-5 人，持续部署** + +### 分支模型 + +``` +main (受保护，PR 合并，自动部署到生产) +├─ feature/评查详情页 +├─ fix/登录闪退 +└─ hotfix/支付失败 (紧急修生产) +``` + +**核心思想**： +- main 永远可发布 +- feature 分支**生命周期短**（1-3 天最佳，最长 1 周） +- 所有改动走 PR + +### 日常工作流（每个任务都按这个跑） + +```bash +# 1. 开分支前先同步 main +git checkout main +git pull --rebase + +# 2. 创建任务分支（任务命名，不要带人名） +git checkout -b feature/评查详情页 + +# 3. 开发，频繁小步提交，频繁推送 +git add . +git commit -m "feat(评查): 新增详情页骨架" +git push -u origin feature/评查详情页 + +# 4. 推送前同步 main，避免冲突堆积 +git fetch origin +git rebase origin/main +# 解决冲突 → 本地测试通过 + +# 5. 推送（rebase 后需要 force-with-lease） +git push --force-with-lease + +# 6. 提 PR → Review → 合并 → 自动部署 + +# 7. 合并后清理本地 +git checkout main +git pull +git branch -d feature/评查详情页 +``` + +### PR 流程规范 + +**提 PR 时**： +- 标题：`(): <简短描述>`，例：`feat(评查): 新增详情页` +- 描述模板： + ```markdown + ## 改动内容 + - ... + + ## 测试方法 + - ... + + ## 截图（如有 UI 改动） + + ## 关联 Issue + Closes #123 + ``` +- 自检：lint 通过、测试通过、不含调试代码 + +**Review 时**： +- 24 小时内响应 +- 给具体可操作的反馈，不要"建议优化一下" +- Approve 前自己心里跑一遍代码逻辑 + +### 紧急修复（hotfix） + +```bash +git checkout main && git pull +git checkout -b hotfix/支付失败 +# 修 → 测 → push → PR → 紧急合并 → 立即部署 +``` + +### 仓库一次性配置（落地关键） + +**main 分支保护（GitHub/GitLab 后台）**： +- ✅ 禁止直接 push +- ✅ 禁止 force push +- ✅ 必须 PR + ≥1 Approve +- ✅ 必须通过 CI（lint + test） +- ✅ 合并后自动删除分支 + +--- + +## 五、方案 C：大项目工作流（Git Flow Lite） + +**适用：6+ 人 / 多版本并存 / 有 QA 测试期** + +### 分支模型 + +``` +main (生产版本，受保护，只接受来自 release/hotfix 的合并) +└─ develop (开发主线，受保护，所有 feature 合并到这里) + ├─ feature/评查详情页 (个人功能分支) + ├─ feature/登录优化 + ├─ release/v2.3.0 (准备发布的快照分支) + └─ hotfix/支付失败 (从 main 拉，修完合并回 main + develop) +``` + +### 各分支角色 + +| 分支 | 来源 | 合并去向 | 生命周期 | +|------|------|---------|---------| +| `main` | 永久 | — | 永久 | +| `develop` | 永久 | — | 永久 | +| `feature/*` | develop | develop | 任务期 | +| `release/*` | develop | main + develop | 发布周期 | +| `hotfix/*` | main | main + develop | 紧急修复 | + +### develop 的角色 + +- **开发主线**：所有 feature 合并到 develop +- **集成测试环境**：QA 在 develop 上测试 +- **不直接发布**：发布前从 develop 拉 release 分支封板 + +### release 分支：发布前的"封板" + +```bash +# 1. 从 develop 拉 release 分支（功能冻结） +git checkout develop && git pull +git checkout -b release/v2.3.0 + +# 2. 在 release 上只修 bug，不加新功能 +# QA 测试 → 修 bug → 改版本号 → 写 changelog + +# 3. 测试通过，合并到 main 并打 tag +git checkout main && git pull +git merge --no-ff release/v2.3.0 +git tag -a v2.3.0 -m "release: v2.3.0" +git push --tags + +# 4. 合并回 develop（带上 release 上的 bug 修复） +git checkout develop +git merge --no-ff release/v2.3.0 +git push + +# 5. 删除 release 分支 +git branch -d release/v2.3.0 +``` + +### hotfix 分支：紧急修生产 + +```bash +# 1. 从 main 拉 hotfix 分支 +git checkout main && git pull +git checkout -b hotfix/支付失败 + +# 2. 修 → 测 → 提 PR + +# 3. 合并到 main 并打 patch tag +git checkout main +git merge --no-ff hotfix/支付失败 +git tag -a v2.3.1 -m "hotfix: 修复支付失败" +git push --tags + +# 4. 必须合并回 develop！否则下次发布 bug 复发 +git checkout develop +git merge --no-ff hotfix/支付失败 +git push + +# 5. 删除 hotfix 分支 +git branch -d hotfix/支付失败 +``` + +### 日常 feature 工作流 + +```bash +# 从 develop（不是 main！）开分支 +git checkout develop && git pull --rebase +git checkout -b feature/评查详情页 + +# 开发 → 推送 → PR 到 develop（不是 main！） +git push -u origin feature/评查详情页 +# 提 PR：feature/评查详情页 → develop +``` + +### 大项目仓库配置 + +- **main**：禁止 push、禁止 force push、只接受 release/hotfix 合并、必须 ≥2 Approve +- **develop**：禁止 push、必须 PR + ≥1 Approve、必须通过 CI +- **CI 要求**：lint + 单元测试 + 集成测试 + 静态分析 + +--- + +## 六、为什么禁止使用个人分支【专题】 + +### 反模式（团队中常见错误） + +``` +❌ feature/评查详情页-zhangsan +❌ zhangsan-dev +❌ feature/zhangsan/xxx +❌ 张三的分支 +❌ wy-dev / wy-test +``` + +### 5 大问题 + +#### 1️⃣ 强化"个人所有权"，破坏协作精神 + +分支带人名 → 同事潜意识"这是张三的地盘，我别动" → 协作变独行 → 出问题没人帮看。 + +**好的协作是任务驱动**：分支属于任务，谁有空谁接手。 + +#### 2️⃣ 离职、休假、轮岗时难交接 + +**真实场景**： +> 张三休假一周，留下 5 条 `feature/xxx-zhangsan` 分支。李四接手时是改名（影响远程）、还是新开 `feature/xxx-lisi`（造成分裂）？怎么处理都别扭。 + +如果分支叫 `feature/评查详情页`，李四直接 `git checkout` 接着干，**零摩擦**。 + +#### 3️⃣ 分支语义模糊，看名字不知道做什么 + +`zhangsan-dev` 这条分支在做啥？ +- 评查模块？ +- 登录优化？ +- 性能调优？ +- 还是张三的"我啥都往这塞"草稿？ + +**好的分支名 = 一句话任务说明**，看到 `feature/评查详情页` 立刻明白。 + +#### 4️⃣ 容易堆积成"杂物分支" + +人名分支没有边界，张三会把"评查详情页 + 顺手优化的登录 + 试验性能改动"都塞进 `feature/zhangsan-dev`。结果： +- PR 改动巨大，没人能 Review +- 一个 bug 拖延了三个功能 +- 不能单独 revert 某项改动 + +**任务分支天然有边界**：`feature/评查详情页` 只装评查相关改动，超出范围就该开新分支。 + +#### 5️⃣ 多人协作同一功能时命名混乱 + +**真实场景**： +> 评查详情页很大，张三李四王五一起做。分支会变成： +> - `feature/评查详情页-zhangsan` +> - `feature/评查详情页-lisi` +> - `feature/评查详情页-wangwu` +> +> 三人改动如何同步？合并顺序？谁先合谁后合？混乱。 + +**正确做法**：开一条父分支 `feature/评查详情页`，每人开**子任务分支**： +- `feature/评查详情页-头部` +- `feature/评查详情页-列表` +- `feature/评查详情页-筛选` + +子任务都合并回父分支，最后父分支整体合到 develop。 + +--- + +### 真实案例：人名分支造成的事故 + +**案例 1：交接黑洞** +> 某团队张三离职前留下 `zhangsan/refactor-payment` 分支，里面有他重构支付的工作（约 60% 完成）。半年后团队想接着做，发现：分支与 main 已差异巨大，无法 rebase；分支内 commit 信息全是"WIP"，没人知道每个 commit 在做什么；最终只能放弃，**重做**。 + +**案例 2：紧急修复无法落地** +> 周五晚上生产支付功能挂了。值班发现修复需要改一个文件，但该文件在 `feature/支付重构-zhangsan` 分支上张三已改了一周。要么强行修生产（与张三的分支冲突会变大），要么等张三周一来。最终用了一个肮脏的 hotfix，**技术债累计**。 + +**案例 3：PR Review 无人能审** +> `feature/zhangsan-dev` 一个 PR 改了 2000 行，包含评查、登录、列表、性能 4 个不相关的改动。Review 同事看了一上午放弃，盖章通过。两天后线上 bug，**无法定位是哪个改动引起的**。 + +--- + +### 正确做法 + +#### ✅ 任务命名规范 + +``` +feature/<任务描述> feature/评查详情页 +feature/-<描述> feature/PROJ-123-评查详情页 +fix/ fix/登录闪退 +hotfix/<紧急bug> hotfix/支付失败 +refactor/<范围> refactor/数据层 +``` + +#### ✅ 任务归属体现在 PR/Issue Assignee + +谁负责这个任务 → 在 **PR/Issue 的 Assignee** 字段标记，不要污染分支名。 + +#### ✅ 分支只与「任务/目标」关联 + +看到 `feature/评查详情页` 应当能立刻回答： +- 这条分支在做什么 +- 合并目标是什么 +- 何时该删除 + +--- + +### 例外：可以带人名的场景 + +**仅限以下短期、可丢弃的情况**： + +#### 1. 个人探索/草稿分支（明确不打算合并） + +``` +zhangsan/试个性能方案 +zhangsan/sandbox +``` + +用斜线作为"个人 sandbox 命名空间"前缀，与正式 feature 分支区分。 + +#### 2. Fork 工作流（开源项目常见） + +你 fork 仓库到自己账号后，分支名随意，提 PR 时再说。这是**远程 fork 模式**，不属于"单仓库协作"场景。 + +#### 3. 多人并行试方案的临时区分 + +``` +feature/评查详情页/方案A-zhangsan +feature/评查详情页/方案B-lisi +``` + +验证完保留赢的方案，删除全部分支，重命名为正式分支。 + +> **核心：这些场景都是短期、明确不污染主流分支的。** + +--- + +## 七、通用规范 + +### 分支命名 + +``` +/ +/- +``` + +**type 类型**： +- `feature/` 新功能 +- `fix/` bug 修复 +- `hotfix/` 紧急修复（生产） +- `refactor/` 重构 +- `docs/` 文档 +- `test/` 测试 +- `chore/` 杂项（依赖升级、配置） +- `release/` 发布分支（仅 Git Flow） + +**示例**： +``` +feature/评查详情页 +feature/PROJ-123-评查详情页 +fix/登录闪退 +hotfix/v2.3-支付失败 +``` + +**禁止**： +- ❌ 人名前缀/后缀 +- ❌ 拼音缩写如 `wy-dev` +- ❌ 没有 type 前缀 +- ❌ 空格、特殊字符 + +### 提交信息（Conventional Commits） + +``` +(): <简短描述> + +<可选详细说明> + +<可选 footer> +``` + +**type**： +- `feat:` 新功能 +- `fix:` 修 bug +- `refactor:` 重构（不改外部行为） +- `perf:` 性能优化 +- `test:` 测试 +- `docs:` 文档 +- `style:` 格式（不改代码逻辑） +- `chore:` 杂项 +- `build:` 构建相关 +- `ci:` CI 相关 + +**示例**： +``` +feat(评查): 新增详情页骨架屏 + +- 添加加载占位组件 +- 实现进入动画 +- 适配移动端 + +Closes #123 +``` + +**禁止**： +- ❌ `update`、`fix`、`修改`、`123` +- ❌ 一个 commit 改 N 个不相关的事 +- ❌ `WIP`（合并前应 squash 掉） + +### 标签（Tag）规范 + +**版本标签**遵循 [SemVer](https://semver.org/)： +``` +v1.2.3 正式版 +v1.2.3-beta.1 beta 版 +v1.2.3-rc.1 发布候选版 +``` + +**打 tag 时机**： +- 个人项目：每次发布 +- 小项目：每次部署到生产 +- 大项目：release 合并到 main 时 + +### .gitignore 通用模板 + +```gitignore +# 依赖 +node_modules/ +__pycache__/ +*.pyc +venv/ +.venv/ + +# 环境变量（永远不要提交） +.env +.env.* +!.env.example + +# 编辑器 +.vscode/ +.idea/ +*.swp +.DS_Store + +# 构建产物 +dist/ +build/ +*.log +coverage/ + +# 本地配置 +.cache/ +*.local +``` + +--- + +## 八、共享资源协作守则 + +### 通用原则 + +> **凡是会影响他人的操作，事前通知、事中可见、事后可恢复。** + +### 共享资源守则表 + +| 共享资源 | 守则 | +|---------|------| +| 共享开发/测试机 | 默认主分支；切换通知；用完还原；优先 `git worktree` | +| 共享数据库 | 不在共用库做破坏性改动；每人独立 schema；测试用 docker 起本地 | +| 共享 API Key / Secret | 用 secret manager；禁止贴 IM/代码/截图 | +| CI/CD pipeline | 改流水线前通知；不在大家忙时触发昂贵任务 | +| 第三方账号 | SSO + 审计；操作记录可追溯 | +| 部署环境 | 部署窗口公告；锁机制（谁部署谁举手） | + +### 切换共享资源的标准动作 + +``` +1. 群里通知："我要 [操作] [资源]，预计 [时间]" +2. 操作 +3. 用完恢复默认状态 +4. 群里告知："已恢复" +``` + +### 优先用隔离手段，而不是切换 + +```bash +# git worktree：在不同目录跑不同分支，主目录不动 +git worktree add ../preview-xxx feature/xxx +git worktree remove ../preview-xxx + +# Docker：每个分支起独立容器 +docker compose -p preview-xxx up +``` + +### 终极方案：消除"共享"本身 + +- 每个 PR 自动起一个独立预览环境（Vercel / Netlify / K8s preview） +- 每人独立数据库 schema（自动迁移） +- 每人独立云资源（云厂商沙盒账号） + +> **根本不需要"共享调试机"。** + +--- + +## 九、常见问题 FAQ + +### Q1：合并前应该用 merge 还是 rebase？ + +| 场景 | 推荐 | +|------|------| +| 个人 feature 分支同步 main | `git rebase`（历史干净） | +| 多人协作的分支 | `git merge`（不重写他人 commit） | +| 合并 PR 到 main | **Squash and merge**（main 历史干净） | +| 已推送的分支 | 谨慎使用 rebase，需 `--force-with-lease` | + +### Q2：feature 分支多久合并一次？ + +- **理想**：1-3 天 +- **可接受**：1 周 +- **超过 1 周必须警惕**：分支过大 → 拆分；同步频率提高（每天 rebase main） + +### Q3：commit 太多太乱怎么办？ + +合并到主分支前 squash 整理： +```bash +git rebase -i HEAD~5 +# 把第二个开始的 pick 改成 squash 或 fixup +``` + +或在 PR 合并时选择 **"Squash and merge"**。 + +### Q4：误推了敏感信息（密钥）怎么办？ + +1. **立即作废**这个密钥（去后台重新生成） +2. 用 `git filter-repo` 或 BFG 清除历史 +3. **强制推送**所有分支 +4. **通知所有人**重新 clone + +预防： +- `.gitignore` 配齐 `.env` +- 用 secret manager +- 装 `git-secrets` / `gitleaks` pre-commit 钩子 + +### Q5：rebase 时遇到冲突怎么办？ + +```bash +# 1. 解决冲突文件 +# 2. git add <冲突文件> +# 3. git rebase --continue + +# 实在搞不定就放弃 +git rebase --abort +``` + +### Q6：误删了分支怎么恢复？ + +```bash +# 找到最后一次 commit +git reflog + +# 恢复 +git checkout -b feature/xxx +``` + +### Q7：`git push` 被拒（rejected） + +意味着远程分支有新 commit 你本地没有。 + +```bash +# 同步后再推 +git pull --rebase +git push + +# 如果是 rebase 后推自己的分支 +git push --force-with-lease # 注意是 --force-with-lease，不是 --force +``` + +> ⚠️ **永远不要对 main / develop 用 `--force`！** + +### Q8：什么时候删分支？ + +- 合并到主干后**立即删除**远程和本地 +- GitHub/GitLab 可设置"合并后自动删除" +- 本地：`git branch -d feature/xxx` + +### Q9：长期废弃的分支能直接删吗？ + +先 `git log feature/xxx` 看是否有未合并 commit。若有： +- 询问作者是否还需要 +- 或导出 patch：`git format-patch main..feature/xxx` + +### Q10：Reviewer 检查清单 + +审视 PR 是否值得合并： + +- [ ] 分支命名规范，无人名 +- [ ] 改动只做一件事，与 PR 标题一致 +- [ ] 提交信息规范 +- [ ] 测试覆盖关键路径 +- [ ] CI 全绿 +- [ ] 无调试代码（console.log、注释掉的代码） +- [ ] 无敏感信息（API Key、密码） +- [ ] 文档/注释 同步更新 +- [ ] 跑过本地测试 + +--- + +## 十、配套工具与一键配置 + +### 全员 Git 配置（每台机器跑一次） + +```bash +# pull 默认 rebase（避免无意义 merge commit） +git config --global pull.rebase true + +# push 自动追踪上游 +git config --global push.autoSetupRemote true + +# rebase 自动 stash +git config --global rebase.autoStash true + +# 默认分支名 +git config --global init.defaultBranch main + +# 换行符处理 +git config --global core.autocrlf input # Mac/Linux +git config --global core.autocrlf true # Windows +``` + +### 推荐工具 + +| 类型 | 工具 | 作用 | +|------|------|------| +| 提交检查 | husky + lint-staged | commit 前自动跑 lint/format | +| 提交规范 | commitlint | 强制 Conventional Commits | +| 密钥扫描 | gitleaks / git-secrets | 防止推送密钥 | +| 图形化客户端 | Fork / SourceTree / GitKraken | 新人友好 | +| CLI 增强 | gh / lazygit | 命令行高效操作 | +| CI/CD | GitHub Actions / GitLab CI | 自动测试和部署 | +| PR 预览 | Vercel / Netlify | 每 PR 一个独立环境 | + +### 一键安装 husky + lint-staged + commitlint（Node 项目） + +```bash +bun add -D husky lint-staged @commitlint/cli @commitlint/config-conventional + +# 初始化 husky +bunx husky init + +# 配置 commitlint +echo "export default { extends: ['@commitlint/config-conventional'] }" > commitlint.config.mjs + +# 添加 hooks +echo "bunx commitlint --edit \$1" > .husky/commit-msg +echo "bunx lint-staged" > .husky/pre-commit +``` + +`package.json` 添加： + +```json +{ + "lint-staged": { + "*.{js,ts,tsx}": ["eslint --fix", "prettier --write"] + } +} +``` + +--- + +## 附录：决策速查表 + +### 我该开什么类型的分支？ + +| 我要做的事 | 分支前缀 | 例子 | +|----------|---------|------| +| 新功能 | `feature/` | `feature/评查详情页` | +| 修 bug | `fix/` | `fix/登录闪退` | +| 紧急修生产 | `hotfix/` | `hotfix/支付失败` | +| 重构 | `refactor/` | `refactor/数据层` | +| 写文档 | `docs/` | `docs/api文档` | +| 升级依赖 | `chore/` | `chore/升级react18` | + +### 我该写什么 commit type？ + +| 我做了什么 | type | +|----------|------| +| 写了新功能 | `feat` | +| 修 bug | `fix` | +| 重构（不改行为） | `refactor` | +| 性能优化 | `perf` | +| 改文档 | `docs` | +| 改格式（空格、缩进） | `style` | +| 加测试 | `test` | +| 杂项 | `chore` | + +### 我该用哪个工作流？ + +``` +1 人 → 方案 A（main + 偶尔 feature） +2-5 人 → 方案 B（GitHub Flow，main + feature + PR） +6+ 人 / 多版本 → 方案 C（Git Flow Lite，main + develop + ...） +``` + +--- + +## 附录：一页纸速查（打印贴墙） + +``` +┌─────────────────────────────────────────────────────────┐ +│ 团队 Git 协作 · 一页纸速查 │ +├─────────────────────────────────────────────────────────┤ +│ 1. 永远从最新主干开任务分支 │ +│ git checkout main && git pull --rebase │ +│ git checkout -b feature/任务描述 │ +│ │ +│ 2. 分支命名 = 任务，不带人名！ │ +│ ✅ feature/评查详情页 │ +│ ❌ feature/评查详情页-zhangsan │ +│ │ +│ 3. commit 写人话 │ +│ ✅ feat(评查): 新增详情页骨架 │ +│ ❌ update │ +│ │ +│ 4. 推送前先同步 │ +│ git fetch && git rebase origin/main │ +│ │ +│ 5. 主分支 = PR 入口，禁止直推 │ +│ │ +│ 6. 共享资源 = 通知 → 操作 → 还原 → 告知 │ +│ │ +│ 7. 永远不对 main/develop --force │ +│ │ +│ 8. 合并后立刻删分支 │ +└─────────────────────────────────────────────────────────┘ +``` + +--- + diff --git a/docs/接口/RAG聊天接口.md b/docs/接口/RAG聊天接口.md new file mode 100644 index 0000000..b2a14a7 --- /dev/null +++ b/docs/接口/RAG聊天接口.md @@ -0,0 +1,493 @@ +# RAG 聊天接口 + +> 最后整理：2026-05-07 +> 对应后端：`fastapi_modules/fastapi_leaudit/controllers/ragChatController.py` +> 统一前缀：`/api/v3/rag` + +## 1. 目标与范围 + +本组接口用于替代旧的 `dify_chat/*` 对话代理，直接提供自有 RAG 聊天能力。 + +当前已落地能力： + +- 获取当前用户可见的 RAG 应用 +- 获取默认 RAG 应用 +- 获取当前用户可见知识库 +- 获取聊天页面参数 +- 发起流式对话 +- 查询会话列表 +- 查询会话消息 +- 重命名会话 +- 删除会话 +- 消息反馈 + +当前不在本文档范围内： + +- 知识库 CRUD 管理 +- 文档切分 / 入库任务 +- Chroma 集合构建脚本 + +## 2. 鉴权与权限 + +### 2.1 鉴权方式 + +所有接口都要求请求头带： + +```http +Authorization: Bearer +``` + +JWT 解析逻辑见： + +- `fastapi_common/fastapi_common_security/security.py` +- `fastapi_common/fastapi_common_security/jwtService.py` + +JWT payload 至少会被后端消费这些字段： + +- `user_id` +- `username` +- `area` +- `user_role` +- `type`，必须为 `access` + +### 2.2 权限键 + +| 接口 | 权限 | +|------|------| +| `GET /api/v3/rag/apps` | `rag:app:read` | +| `GET /api/v3/rag/apps/default` | `rag:app:read` | +| `GET /api/v3/rag/datasets/my` | `rag:dataset:read` | +| `GET /api/v3/rag/chat/parameters` | `rag:chat:use` 或 `rag:app:read` 其一即可 | +| `POST /api/v3/rag/chat/messages` | `rag:chat:use` | +| `GET /api/v3/rag/chat/conversations` | `rag:conversation:read` | +| `GET /api/v3/rag/chat/conversations/{ConversationId}/messages` | `rag:conversation:read` | +| `PATCH /api/v3/rag/chat/conversations/{ConversationId}` | `rag:conversation:update` | +| `DELETE /api/v3/rag/chat/conversations/{ConversationId}` | `rag:conversation:delete` | +| `POST /api/v3/rag/chat/messages/{MessageId}/feedback` | `rag:message:feedback` | + +说明： + +- 权限检查使用 `HasAnyPermission`，即“列表中的任一权限命中即可通过”。 +- 具体实现见 `fastapi_modules/fastapi_leaudit/services/impl/permissionServiceImpl.py`。 + +## 3. 通用返回格式 + +除流式接口外，统一返回： + +```json +{ + "code": 200, + "msg": "success", + "data": {} +} +``` + +权限不足时通常返回： + +```json +{ + "code": 403, + "msg": "当前用户没有对应权限", + "data": null +} +``` + +## 4. 接口列表 + +### 4.1 获取可见应用列表 + +`GET /api/v3/rag/apps` + +用途： + +- 聊天页面加载应用下拉 +- 根据地区 / 省级角色做应用可见性过滤 + +请求参数：无 + +成功响应示例： + +```json +{ + "code": 200, + "msg": "success", + "data": { + "data": [ + { + "appId": "1", + "appName": "法务问答", + "description": "默认烟草法务知识问答", + "isDefault": true + } + ], + "total": 1 + } +} +``` + +可见性规则： + +- `provincial_admin` 可见全部启用应用 +- 其他角色仅可见： + - `rag_chat_app.area = 用户 area` + - `rag_chat_app.area = '省级'` + - `rag_chat_app.area = ''` + - 或关联数据集 `rag_dataset.is_public = true` + +### 4.2 获取默认应用 + +`GET /api/v3/rag/apps/default` + +用途： + +- 聊天页初始化默认应用 + +请求参数：无 + +成功响应示例： + +```json +{ + "code": 200, + "msg": "success", + "data": { + "appId": "1", + "appName": "法务问答", + "description": "默认烟草法务知识问答", + "isDefault": true + } +} +``` + +补充说明： + +- 如果没有显式默认应用，会退回到当前用户可见的第一条应用。 +- 如果完全没有可见应用，返回 `data: null`。 + +### 4.3 获取当前用户可见知识库 + +`GET /api/v3/rag/datasets/my` + +用途： + +- 聊天页展示“当前可用知识库”时使用 + +请求参数：无 + +成功响应示例： + +```json +{ + "code": 200, + "msg": "success", + "data": { + "data": [ + { + "id": 1, + "name": "广东烟草法规库", + "description": "省级法规与制度", + "area": "省级", + "isPublic": true, + "isDefault": true, + "documentCount": 120, + "totalChunks": 8345, + "status": 1 + } + ], + "total": 1 + } +} +``` + +### 4.4 获取聊天参数 + +`GET /api/v3/rag/chat/parameters` + +查询参数： + +| 参数 | 类型 | 必填 | 说明 | +|------|------|------|------| +| `appId` | int | 否 | 指定应用 ID；不传则取默认应用 | + +用途： + +- 初始化开场白 +- 初始化建议问题 +- 初始化上传能力配置 + +成功响应示例： + +```json +{ + "code": 200, + "msg": "success", + "data": { + "openingStatement": "你好，我可以帮你解答烟草行业法务问题。", + "suggestedQuestions": [ + "烟草专卖许可延续的法定条件是什么？", + "行政处罚文书审查重点有哪些？" + ], + "userInputForm": [], + "fileUpload": { + "image": { + "enabled": false + } + } + } +} +``` + +### 4.5 发起流式对话 + +`POST /api/v3/rag/chat/messages` + +请求体： + +```json +{ + "query": "烟草专卖许可证延续申请的审查要点是什么？", + "conversationId": null, + "appId": 1 +} +``` + +字段说明： + +| 字段 | 类型 | 必填 | 说明 | +|------|------|------|------| +| `query` | string | 是 | 用户问题，不能为空 | +| `conversationId` | string \| null | 否 | 会话 ID；新对话可传 `null` 或不传 | +| `appId` | int \| null | 否 | 应用 ID；不传则自动回退默认应用 | + +返回类型： + +- `text/event-stream` + +SSE 事件 1：流式正文片段 + +```text +data: {"event":"message","task_id":"...","message_id":"...","conversation_id":"...","answer":"第一段内容","created_at":1746580000} + +``` + +SSE 事件 2：流式结束 + +```text +data: {"event":"message_end","task_id":"...","message_id":"...","conversation_id":"...","metadata":{"usage":{"total_tokens":1234},"retriever_resources":[...],"suggested_questions":["问题1","问题2"]}} + +``` + +SSE 事件 3：模型异常 + +```text +data: {"event":"error","task_id":"...","message_id":"...","code":"llm_error","message":"..."} + +``` + +服务端行为说明： + +- 若 `conversationId` 为空，会自动创建新会话 +- 会先落一条 `role = user` 消息，再流式生成回答 +- 流结束后会落一条 `role = assistant` 消息 +- 若命中知识库，会把引用结果写入 `sources / metadata` +- 会根据对话内容追加 `suggested_questions` + +### 4.6 获取会话列表 + +`GET /api/v3/rag/chat/conversations` + +查询参数： + +| 参数 | 类型 | 必填 | 默认值 | 说明 | +|------|------|------|--------|------| +| `appId` | int | 否 | - | 按应用过滤 | +| `page` | int | 否 | `1` | 页码，从 1 开始 | +| `pageSize` | int | 否 | `20` | 每页数量，最大 100 | + +成功响应示例： + +```json +{ + "code": 200, + "msg": "success", + "data": { + "data": [ + { + "id": "b17d3b0b-xxxx-xxxx", + "name": "新对话", + "introduction": "", + "createdAt": 1746580000, + "updatedAt": 1746580066 + } + ], + "hasMore": false, + "limit": 20 + } +} +``` + +### 4.7 获取会话消息 + +`GET /api/v3/rag/chat/conversations/{ConversationId}/messages` + +路径参数： + +| 参数 | 类型 | 说明 | +|------|------|------| +| `ConversationId` | string | 会话 ID | + +查询参数： + +| 参数 | 类型 | 必填 | 默认值 | +|------|------|------|--------| +| `page` | int | 否 | `1` | +| `pageSize` | int | 否 | `20` | + +成功响应示例： + +```json +{ + "code": 200, + "msg": "success", + "data": { + "data": [ + { + "id": "assistant-message-id", + "conversationId": "b17d3b0b-xxxx-xxxx", + "query": "烟草专卖许可证延续申请的审查要点是什么？", + "answer": "先核对主体资格，再核对经营条件……", + "feedback": { + "rating": "like" + }, + "retrieverResources": [ + { + "position": 1, + "dataset_id": "1", + "dataset_name": "广东烟草法规库", + "document_id": "12", + "document_name": "行政许可审查规范.pdf", + "segment_id": "chunk-1", + "score": 0.9132 + } + ], + "createdAt": 1746580000 + } + ], + "hasMore": false, + "limit": 20 + } +} +``` + +说明： + +- 返回结构是按“问答对”聚合后的结果，不是底层 `rag_message` 原始逐条结果。 + +### 4.8 重命名会话 + +`PATCH /api/v3/rag/chat/conversations/{ConversationId}` + +请求体： + +```json +{ + "name": "许可证延续审查要点" +} +``` + +成功响应示例： + +```json +{ + "code": 200, + "msg": "success", + "data": { + "result": "success", + "name": "许可证延续审查要点" + } +} +``` + +### 4.9 删除会话 + +`DELETE /api/v3/rag/chat/conversations/{ConversationId}` + +成功响应示例： + +```json +{ + "code": 200, + "msg": "success", + "data": { + "result": "success" + } +} +``` + +说明： + +- 逻辑删除，实际是设置 `rag_conversation.deleted_at` + +### 4.10 消息反馈 + +`POST /api/v3/rag/chat/messages/{MessageId}/feedback` + +请求体： + +```json +{ + "rating": "like" +} +``` + +可选值： + +- `like` +- `dislike` +- `null` + +成功响应示例： + +```json +{ + "code": 200, + "msg": "success", + "data": { + "result": "success" + } +} +``` + +## 5. 数据表对应关系 + +| 表 | 用途 | +|----|------| +| `rag_dataset` | 知识库定义、地区可见性、检索参数 | +| `rag_document` | 知识库文档、启停状态、命中次数 | +| `rag_chat_app` | 聊天应用配置、默认应用、应用绑定知识库 | +| `rag_conversation` | 用户会话 | +| `rag_message` | 底层消息落库，包含引用与反馈 | + +## 6. 当前实现约束 + +- 当前只提供 `GET /datasets/my`，不提供知识库管理 CRUD +- `fileUpload.image.enabled` 固定为 `false` +- 检索依赖 Chroma；Chroma 不可用时，接口仍可回答，但会退化成无知识库上下文 +- 建议问题 `suggestedQuestions` 由二次模型调用生成，失败时会降级为空数组 + +## 7. 联调建议 + +最小联调顺序： + +1. `GET /api/v3/rag/apps` +2. `GET /api/v3/rag/apps/default` +3. `GET /api/v3/rag/chat/parameters` +4. `POST /api/v3/rag/chat/messages` +5. `GET /api/v3/rag/chat/conversations` +6. `GET /api/v3/rag/chat/conversations/{ConversationId}/messages` + +优先检查： + +- 401：JWT 是否有效 +- 403：`rag:*` 权限是否已写入并分配 +- 500：LLM、数据库、Chroma、知识库配置是否可用 diff --git a/docs/接口/README.md b/docs/接口/README.md index 36d5792..e1f3541 100644 --- a/docs/接口/README.md +++ b/docs/接口/README.md @@ -17,6 +17,7 @@ |------|------|------| | 首页入口 / 菜单 | `入口模块绑定最终设计方案.md` | 入口模块、文档类型、规则链路绑定模型 | | 文档上传 / 列表 / 评查 | `文档上传与列表接口分析.md` | 上传、列表、详情、更新、删除、评查触发、数据隔离 | +| RAG 聊天 | `RAG聊天接口.md` | `/api/v3/rag/*` 自有聊天接口、SSE、权限、表结构映射 | | 文档类型 / 评查组 | `评查点分组目标结构与迁移方案.md` | 文档类型、一级分组、二级分组、规则集与迁移口径 | | 评查点分组迁移 | `评查点分组目标结构与迁移方案.md` | 新老分组结构对齐方案 | | 评查点分组迁移 | `评查点分组迁移执行前检查清单.md` | 正式迁移前检查项 | diff --git a/fastapi_common/fastapi_common_security/jwtService.py b/fastapi_common/fastapi_common_security/jwtService.py index 5518650..aaa438f 100644 --- a/fastapi_common/fastapi_common_security/jwtService.py +++ b/fastapi_common/fastapi_common_security/jwtService.py @@ -61,6 +61,8 @@ class JwtService: jti = str(uuid.uuid4()) # Access Token + # Token 只保留鉴权链路真正需要的最小字段，避免省局/管理员权限过多时 + # 把 permissions / roles 全塞进 JWT，最终导致前端 Cookie Session 超过 4KB。 accessPayload = { "jti": jti, "user_id": userId, @@ -68,8 +70,6 @@ class JwtService: "nick_name": nickName, "ou_id": ouId, "ou_name": ouName, - "roles": roles or [], - "permissions": permissions or [], "area": area, "user_role": userRole, "iat": now, diff --git a/fastapi_modules/fastapi_leaudit/controllers/ragChatController.py b/fastapi_modules/fastapi_leaudit/controllers/ragChatController.py new file mode 100644 index 0000000..7c79d23 --- /dev/null +++ b/fastapi_modules/fastapi_leaudit/controllers/ragChatController.py @@ -0,0 +1,173 @@ +from __future__ import annotations + +from typing import Any + +from fastapi import Depends, Query +from fastapi.responses import JSONResponse, StreamingResponse + +from fastapi_common.fastapi_common_security.security import verify_access_token +from fastapi_common.fastapi_common_web.controller import BaseController +from fastapi_common.fastapi_common_web.domain.responses import Result + +from fastapi_modules.fastapi_leaudit.domian.Dto.ragChatDto import ( + RagConversationRenameDTO, + RagChatSendMessageDTO, + RagMessageFeedbackDTO, +) +from fastapi_modules.fastapi_leaudit.domian.vo.ragChatVo import ( + RagAppParametersVO, + RagChatAppListVO, + RagChatAppVO, + RagConversationPageVO, + RagConversationRenameVO, + RagMessagePageVO, + RagOperationResultVO, +) +from fastapi_modules.fastapi_leaudit.domian.vo.ragDatasetVo import RagDatasetPageVO +from fastapi_modules.fastapi_leaudit.services.impl.permissionServiceImpl import PermissionServiceImpl +from fastapi_modules.fastapi_leaudit.services.impl.ragChatServiceImpl import RagChatServiceImpl +from fastapi_modules.fastapi_leaudit.services.impl.ragDatasetServiceImpl import RagDatasetServiceImpl +from fastapi_modules.fastapi_leaudit.services.permissionService import IPermissionService +from fastapi_modules.fastapi_leaudit.services.ragChatService import IRagChatService +from fastapi_modules.fastapi_leaudit.services.ragDatasetService import IRagDatasetService + + +class RagChatController(BaseController): + _PERMISSIONS = { + "chat_use": "rag:chat:use", + "conversation_read": "rag:conversation:read", + "conversation_update": "rag:conversation:update", + "conversation_delete": "rag:conversation:delete", + "message_feedback": "rag:message:feedback", + "app_read": "rag:app:read", + "dataset_read": "rag:dataset:read", + } + + def __init__(self): + super().__init__(prefix="/v3/rag", tags=["RAG 聊天"]) + self.RagChatService: IRagChatService = RagChatServiceImpl() + self.RagDatasetService: IRagDatasetService = RagDatasetServiceImpl() + self.PermissionService: IPermissionService = PermissionServiceImpl() + + @self.router.get("/apps", response_model=Result[RagChatAppListVO]) + async def GetApps(payload: dict[str, Any] = Depends(verify_access_token)): + if not await self._check_permission(int(payload["user_id"]), [self._PERMISSIONS["app_read"]]): + return JSONResponse(status_code=403, content={"code": 403, "msg": "当前用户没有查看聊天应用权限", "data": None}) + data = await self.RagChatService.GetApps( + CurrentUserId=int(payload["user_id"]), + UserArea=payload.get("area"), + UserRole=payload.get("user_role"), + ) + return Result.success(data=data) + + @self.router.get("/apps/default", response_model=Result[RagChatAppVO | None]) + async def GetDefaultApp(payload: dict[str, Any] = Depends(verify_access_token)): + if not await self._check_permission(int(payload["user_id"]), [self._PERMISSIONS["app_read"]]): + return JSONResponse(status_code=403, content={"code": 403, "msg": "当前用户没有查看默认聊天应用权限", "data": None}) + data = await self.RagChatService.GetDefaultApp( + CurrentUserId=int(payload["user_id"]), + UserArea=payload.get("area"), + UserRole=payload.get("user_role"), + ) + return Result.success(data=data) + + @self.router.get("/datasets/my", response_model=Result[RagDatasetPageVO]) + async def GetMyDatasets(payload: dict[str, Any] = Depends(verify_access_token)): + if not await self._check_permission(int(payload["user_id"]), [self._PERMISSIONS["dataset_read"]]): + return JSONResponse(status_code=403, content={"code": 403, "msg": "当前用户没有查看知识库权限", "data": None}) + data = await self.RagDatasetService.GetMyDatasets( + CurrentUserId=int(payload["user_id"]), + UserArea=payload.get("area"), + UserRole=payload.get("user_role"), + ) + return Result.success(data=data) + + @self.router.get("/chat/parameters", response_model=Result[RagAppParametersVO]) + async def GetAppParameters( + appId: int | None = Query(None, description="聊天应用ID"), + payload: dict[str, Any] = Depends(verify_access_token), + ): + if not await self._check_permission(int(payload["user_id"]), [self._PERMISSIONS["chat_use"], self._PERMISSIONS["app_read"]]): + return JSONResponse(status_code=403, content={"code": 403, "msg": "当前用户没有查看聊天配置权限", "data": None}) + data = await self.RagChatService.GetAppParameters( + CurrentUserId=int(payload["user_id"]), + UserArea=payload.get("area"), + UserRole=payload.get("user_role"), + AppId=appId, + ) + return Result.success(data=data) + + @self.router.post("/chat/messages") + async def SendMessage(Body: RagChatSendMessageDTO, payload: dict[str, Any] = Depends(verify_access_token)): + if not await self._check_permission(int(payload["user_id"]), [self._PERMISSIONS["chat_use"]]): + return JSONResponse(status_code=403, content={"code": 403, "msg": "当前用户没有使用 RAG 对话权限", "data": None}) + stream = self.RagChatService.SendMessage( + CurrentUserId=int(payload["user_id"]), + UserName=payload.get("username") or str(payload.get("user_id")), + UserArea=payload.get("area"), + UserRole=payload.get("user_role"), + Query=Body.query, + ConversationId=Body.conversationId, + AppId=Body.appId, + ) + return StreamingResponse( + stream, + media_type="text/event-stream", + headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no", "Connection": "keep-alive"}, + ) + + @self.router.get("/chat/conversations", response_model=Result[RagConversationPageVO]) + async def GetConversations( + appId: int | None = Query(None, description="聊天应用ID"), + page: int = Query(1, ge=1), + pageSize: int = Query(20, ge=1, le=100), + payload: dict[str, Any] = Depends(verify_access_token), + ): + if not await self._check_permission(int(payload["user_id"]), [self._PERMISSIONS["conversation_read"]]): + return JSONResponse(status_code=403, content={"code": 403, "msg": "当前用户没有查看聊天会话权限", "data": None}) + data = await self.RagChatService.GetConversations(int(payload["user_id"]), appId, page, pageSize) + return Result.success(data=data) + + @self.router.get("/chat/conversations/{ConversationId}/messages", response_model=Result[RagMessagePageVO]) + async def GetConversationMessages( + ConversationId: str, + page: int = Query(1, ge=1), + pageSize: int = Query(20, ge=1, le=100), + payload: dict[str, Any] = Depends(verify_access_token), + ): + if not await self._check_permission(int(payload["user_id"]), [self._PERMISSIONS["conversation_read"]]): + return JSONResponse(status_code=403, content={"code": 403, "msg": "当前用户没有查看聊天消息权限", "data": None}) + data = await self.RagChatService.GetConversationMessages(int(payload["user_id"]), ConversationId, page, pageSize) + return Result.success(data=data) + + @self.router.patch("/chat/conversations/{ConversationId}", response_model=Result[RagConversationRenameVO]) + async def RenameConversation( + ConversationId: str, + Body: RagConversationRenameDTO, + payload: dict[str, Any] = Depends(verify_access_token), + ): + if not await self._check_permission(int(payload["user_id"]), [self._PERMISSIONS["conversation_update"]]): + return JSONResponse(status_code=403, content={"code": 403, "msg": "当前用户没有修改聊天会话权限", "data": None}) + data = await self.RagChatService.RenameConversation(int(payload["user_id"]), ConversationId, Body) + return Result.success(data=data) + + @self.router.delete("/chat/conversations/{ConversationId}", response_model=Result[RagOperationResultVO]) + async def DeleteConversation(ConversationId: str, payload: dict[str, Any] = Depends(verify_access_token)): + if not await self._check_permission(int(payload["user_id"]), [self._PERMISSIONS["conversation_delete"]]): + return JSONResponse(status_code=403, content={"code": 403, "msg": "当前用户没有删除聊天会话权限", "data": None}) + data = await self.RagChatService.DeleteConversation(int(payload["user_id"]), ConversationId) + return Result.success(data=data) + + @self.router.post("/chat/messages/{MessageId}/feedback", response_model=Result[RagOperationResultVO]) + async def UpdateFeedback( + MessageId: str, + Body: RagMessageFeedbackDTO, + payload: dict[str, Any] = Depends(verify_access_token), + ): + if not await self._check_permission(int(payload["user_id"]), [self._PERMISSIONS["message_feedback"]]): + return JSONResponse(status_code=403, content={"code": 403, "msg": "当前用户没有反馈聊天消息权限", "data": None}) + data = await self.RagChatService.UpdateFeedback(int(payload["user_id"]), MessageId, Body) + return Result.success(data=data) + + async def _check_permission(self, user_id: int, permission_keys: list[str]) -> bool: + return await self.PermissionService.HasAnyPermission(UserId=user_id, PermissionKeys=permission_keys) diff --git a/fastapi_modules/fastapi_leaudit/domian/Dto/ragChatDto.py b/fastapi_modules/fastapi_leaudit/domian/Dto/ragChatDto.py new file mode 100644 index 0000000..12e530b --- /dev/null +++ b/fastapi_modules/fastapi_leaudit/domian/Dto/ragChatDto.py @@ -0,0 +1,15 @@ +from pydantic import BaseModel, Field + + +class RagChatSendMessageDTO(BaseModel): + query: str = Field(..., min_length=1, description="用户问题") + conversationId: str | None = Field(None, description="会话ID") + appId: int | None = Field(None, description="聊天应用ID") + + +class RagConversationRenameDTO(BaseModel): + name: str = Field(..., min_length=1, max_length=500, description="新会话名称") + + +class RagMessageFeedbackDTO(BaseModel): + rating: str | None = Field(None, description="反馈: like/dislike/None") diff --git a/fastapi_modules/fastapi_leaudit/domian/vo/ragChatVo.py b/fastapi_modules/fastapi_leaudit/domian/vo/ragChatVo.py new file mode 100644 index 0000000..ddf8022 --- /dev/null +++ b/fastapi_modules/fastapi_leaudit/domian/vo/ragChatVo.py @@ -0,0 +1,59 @@ +from pydantic import BaseModel, Field + + +class RagChatAppVO(BaseModel): + appId: str = Field(..., description="应用ID") + appName: str = Field(..., description="应用名称") + description: str = Field("", description="应用描述") + isDefault: bool = Field(False, description="是否默认应用") + + +class RagChatAppListVO(BaseModel): + data: list[RagChatAppVO] = Field(default_factory=list) + total: int = Field(0) + + +class RagConversationItemVO(BaseModel): + id: str = Field(..., description="会话ID") + name: str = Field(..., description="会话名称") + introduction: str = Field("", description="会话简介") + createdAt: int = Field(0, description="创建时间戳") + updatedAt: int = Field(0, description="更新时间戳") + + +class RagConversationPageVO(BaseModel): + data: list[RagConversationItemVO] = Field(default_factory=list) + hasMore: bool = Field(False) + limit: int = Field(20) + + +class RagMessageItemVO(BaseModel): + id: str = Field(...) + conversationId: str = Field(...) + query: str = Field(...) + answer: str = Field(...) + feedback: dict | None = Field(None) + retrieverResources: list[dict] | None = Field(None) + createdAt: int = Field(0) + + +class RagMessagePageVO(BaseModel): + data: list[RagMessageItemVO] = Field(default_factory=list) + hasMore: bool = Field(False) + limit: int = Field(20) + + +class RagConversationRenameVO(BaseModel): + result: str = Field("success") + name: str = Field(...) + + +class RagOperationResultVO(BaseModel): + result: str = Field("success") + + +class RagAppParametersVO(BaseModel): + openingStatement: str = Field("", description="开场白") + suggestedQuestions: list[str] = Field(default_factory=list) + userInputForm: list[dict] = Field(default_factory=list) + fileUpload: dict = Field(default_factory=lambda: {"image": {"enabled": False}}) diff --git a/fastapi_modules/fastapi_leaudit/domian/vo/ragDatasetVo.py b/fastapi_modules/fastapi_leaudit/domian/vo/ragDatasetVo.py new file mode 100644 index 0000000..a69d007 --- /dev/null +++ b/fastapi_modules/fastapi_leaudit/domian/vo/ragDatasetVo.py @@ -0,0 +1,18 @@ +from pydantic import BaseModel, Field + + +class RagDatasetItemVO(BaseModel): + id: int = Field(...) + name: str = Field(...) + description: str = Field("") + area: str = Field("") + isPublic: bool = Field(False) + isDefault: bool = Field(False) + documentCount: int = Field(0) + totalChunks: int = Field(0) + status: int = Field(1) + + +class RagDatasetPageVO(BaseModel): + data: list[RagDatasetItemVO] = Field(default_factory=list) + total: int = Field(0) diff --git a/fastapi_modules/fastapi_leaudit/models/__init__.py b/fastapi_modules/fastapi_leaudit/models/__init__.py index 3f52069..afb1cd5 100644 --- a/fastapi_modules/fastapi_leaudit/models/__init__.py +++ b/fastapi_modules/fastapi_leaudit/models/__init__.py @@ -8,6 +8,11 @@ from fastapi_modules.fastapi_leaudit.models.leauditCrossReviewTask import Leaudi from fastapi_modules.fastapi_leaudit.models.leauditCrossReviewTaskDocument import LeauditCrossReviewTaskDocument from fastapi_modules.fastapi_leaudit.models.leauditCrossReviewTaskMember import LeauditCrossReviewTaskMember from fastapi_modules.fastapi_leaudit.models.leauditCrossReviewVote import LeauditCrossReviewVote +from fastapi_modules.fastapi_leaudit.models.leauditRagDataset import LeauditRagDataset +from fastapi_modules.fastapi_leaudit.models.leauditRagDocument import LeauditRagDocument +from fastapi_modules.fastapi_leaudit.models.leauditRagChatApp import LeauditRagChatApp +from fastapi_modules.fastapi_leaudit.models.leauditRagConversation import LeauditRagConversation +from fastapi_modules.fastapi_leaudit.models.leauditRagMessage import LeauditRagMessage __all__ = [ "LeauditDocument", @@ -18,4 +23,9 @@ __all__ = [ "LeauditCrossReviewTaskDocument", "LeauditCrossReviewProposal", "LeauditCrossReviewVote", + "LeauditRagDataset", + "LeauditRagDocument", + "LeauditRagChatApp", + "LeauditRagConversation", + "LeauditRagMessage", ] diff --git a/fastapi_modules/fastapi_leaudit/models/leauditRagChatApp.py b/fastapi_modules/fastapi_leaudit/models/leauditRagChatApp.py new file mode 100644 index 0000000..6bd0417 --- /dev/null +++ b/fastapi_modules/fastapi_leaudit/models/leauditRagChatApp.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +from sqlalchemy import BigInteger, Boolean, Float, Integer, String, Text +from sqlalchemy.orm import Mapped, mapped_column + +from fastapi_common.fastapi_common_web.models import BaseModel + + +class LeauditRagChatApp(BaseModel): + __tablename__ = "rag_chat_app" + + Id: Mapped[int] = mapped_column("id", BigInteger, primary_key=True, autoincrement=True) + name: Mapped[str] = mapped_column(String(255)) + description: Mapped[str] = mapped_column(Text, default="") + area: Mapped[str] = mapped_column(String(50), default="") + datasetId: Mapped[int | None] = mapped_column("dataset_id", BigInteger) + systemPrompt: Mapped[str] = mapped_column("system_prompt", Text, default="") + llmModel: Mapped[str] = mapped_column("llm_model", String(100), default="") + temperature: Mapped[float] = mapped_column(Float, default=0.3) + maxTokens: Mapped[int] = mapped_column("max_tokens", Integer, default=2048) + openingStatement: Mapped[str] = mapped_column("opening_statement", Text, default="") + suggestedQuestions: Mapped[str] = mapped_column("suggested_questions", Text, default="[]") + isDefault: Mapped[bool] = mapped_column("is_default", Boolean, default=False) + sortOrder: Mapped[int] = mapped_column("sort_order", Integer, default=0) + status: Mapped[int] = mapped_column(Integer, default=1) + createdBy: Mapped[int | None] = mapped_column("created_by", BigInteger) + updatedBy: Mapped[int | None] = mapped_column("updated_by", BigInteger) diff --git a/fastapi_modules/fastapi_leaudit/models/leauditRagConversation.py b/fastapi_modules/fastapi_leaudit/models/leauditRagConversation.py new file mode 100644 index 0000000..86c67d9 --- /dev/null +++ b/fastapi_modules/fastapi_leaudit/models/leauditRagConversation.py @@ -0,0 +1,17 @@ +from __future__ import annotations + +from sqlalchemy import BigInteger, String, Text +from sqlalchemy.orm import Mapped, mapped_column + +from fastapi_common.fastapi_common_web.models import BaseModel + + +class LeauditRagConversation(BaseModel): + __tablename__ = "rag_conversation" + + Id: Mapped[int] = mapped_column("id", BigInteger, primary_key=True, autoincrement=True) + conversationId: Mapped[str] = mapped_column("conversation_id", String(100), unique=True) + userId: Mapped[int] = mapped_column("user_id", BigInteger) + appId: Mapped[int | None] = mapped_column("app_id", BigInteger) + name: Mapped[str] = mapped_column(String(500), default="新对话") + introduction: Mapped[str] = mapped_column(Text, default="") diff --git a/fastapi_modules/fastapi_leaudit/models/leauditRagDataset.py b/fastapi_modules/fastapi_leaudit/models/leauditRagDataset.py new file mode 100644 index 0000000..ac1acfd --- /dev/null +++ b/fastapi_modules/fastapi_leaudit/models/leauditRagDataset.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +from sqlalchemy import BigInteger, Boolean, Integer, String, Text +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.orm import Mapped, mapped_column + +from fastapi_common.fastapi_common_web.models import BaseModel + + +class LeauditRagDataset(BaseModel): + __tablename__ = "rag_dataset" + + Id: Mapped[int] = mapped_column("id", BigInteger, primary_key=True, autoincrement=True) + name: Mapped[str] = mapped_column(String(255), comment="知识库名称") + description: Mapped[str] = mapped_column(Text, default="", comment="知识库描述") + area: Mapped[str] = mapped_column(String(50), default="", comment="地区") + isPublic: Mapped[bool] = mapped_column("is_public", Boolean, default=False) + isDefault: Mapped[bool] = mapped_column("is_default", Boolean, default=False) + collectionName: Mapped[str] = mapped_column("collection_name", String(100), unique=True) + embeddingModel: Mapped[str] = mapped_column("embedding_model", String(100), default="text-embedding-v4") + embeddingDim: Mapped[int] = mapped_column("embedding_dim", Integer, default=1024) + chunkMaxSize: Mapped[int] = mapped_column("chunk_max_size", Integer, default=800) + chunkMinSize: Mapped[int] = mapped_column("chunk_min_size", Integer, default=20) + documentCount: Mapped[int] = mapped_column("document_count", Integer, default=0) + totalChunks: Mapped[int] = mapped_column("total_chunks", Integer, default=0) + retrievalModel: Mapped[dict] = mapped_column("retrieval_model", JSONB, default=dict) + sortOrder: Mapped[int] = mapped_column("sort_order", Integer, default=0) + status: Mapped[int] = mapped_column(Integer, default=1) + createdBy: Mapped[int | None] = mapped_column("created_by", BigInteger) + updatedBy: Mapped[int | None] = mapped_column("updated_by", BigInteger) diff --git a/fastapi_modules/fastapi_leaudit/models/leauditRagDocument.py b/fastapi_modules/fastapi_leaudit/models/leauditRagDocument.py new file mode 100644 index 0000000..0658349 --- /dev/null +++ b/fastapi_modules/fastapi_leaudit/models/leauditRagDocument.py @@ -0,0 +1,24 @@ +from __future__ import annotations + +from sqlalchemy import BigInteger, Boolean, Integer, String, Text +from sqlalchemy.orm import Mapped, mapped_column + +from fastapi_common.fastapi_common_web.models import BaseModel + + +class LeauditRagDocument(BaseModel): + __tablename__ = "rag_document" + + Id: Mapped[int] = mapped_column("id", BigInteger, primary_key=True, autoincrement=True) + datasetId: Mapped[int] = mapped_column("dataset_id", BigInteger) + filename: Mapped[str] = mapped_column(String(500)) + originalName: Mapped[str] = mapped_column("original_name", String(500)) + minioPath: Mapped[str] = mapped_column("minio_path", String(1000)) + fileType: Mapped[str] = mapped_column("file_type", String(20)) + fileSize: Mapped[int] = mapped_column("file_size", BigInteger, default=0) + chunkCount: Mapped[int] = mapped_column("chunk_count", Integer, default=0) + indexingStatus: Mapped[str] = mapped_column("indexing_status", String(20), default="pending") + indexingError: Mapped[str | None] = mapped_column("indexing_error", Text) + enabled: Mapped[bool] = mapped_column(Boolean, default=True) + hitCount: Mapped[int] = mapped_column("hit_count", Integer, default=0) + createdBy: Mapped[int | None] = mapped_column("created_by", BigInteger) diff --git a/fastapi_modules/fastapi_leaudit/models/leauditRagMessage.py b/fastapi_modules/fastapi_leaudit/models/leauditRagMessage.py new file mode 100644 index 0000000..ca44918 --- /dev/null +++ b/fastapi_modules/fastapi_leaudit/models/leauditRagMessage.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +from sqlalchemy import BigInteger, String, Text +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.orm import Mapped, mapped_column + +from fastapi_common.fastapi_common_web.models import BaseModel + + +class LeauditRagMessage(BaseModel): + __tablename__ = "rag_message" + + Id: Mapped[int] = mapped_column("id", BigInteger, primary_key=True, autoincrement=True) + messageId: Mapped[str] = mapped_column("message_id", String(100), unique=True) + conversationId: Mapped[str] = mapped_column("conversation_id", String(100)) + role: Mapped[str] = mapped_column(String(20)) + content: Mapped[str] = mapped_column(Text, default="") + sources: Mapped[list] = mapped_column(JSONB, default=list) + metadataJson: Mapped[dict] = mapped_column("metadata", JSONB, default=dict) + feedback: Mapped[str | None] = mapped_column(String(20)) diff --git a/fastapi_modules/fastapi_leaudit/rag_engine/__init__.py b/fastapi_modules/fastapi_leaudit/rag_engine/__init__.py new file mode 100644 index 0000000..f924b9e --- /dev/null +++ b/fastapi_modules/fastapi_leaudit/rag_engine/__init__.py @@ -0,0 +1 @@ +"""RAG 聊天内核兼容层。""" diff --git a/fastapi_modules/fastapi_leaudit/rag_engine/chroma_client.py b/fastapi_modules/fastapi_leaudit/rag_engine/chroma_client.py new file mode 100644 index 0000000..991a34a --- /dev/null +++ b/fastapi_modules/fastapi_leaudit/rag_engine/chroma_client.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +from typing import Any + +from fastapi_modules.fastapi_leaudit.rag_engine.config import RAG_CONFIG + +_instance: Any | None = None + + +def init_chroma() -> Any: + global _instance + if _instance is not None: + return _instance + + import chromadb # lazy import to avoid hard failure before feature is enabled + import chromadb.config + + host = RAG_CONFIG["CHROMA_HOST"] + if host: + token = RAG_CONFIG.get("CHROMA_TOKEN", "") + header = RAG_CONFIG.get("CHROMA_AUTH_HEADER", "X-Chroma-Token") + settings = ( + chromadb.config.Settings( + chroma_client_auth_provider="chromadb.auth.token_authn.TokenAuthClientProvider", + chroma_client_auth_credentials=token, + chroma_auth_token_transport_header=header, + ) + if token + else chromadb.config.Settings() + ) + _instance = chromadb.HttpClient(host=host, port=RAG_CONFIG["CHROMA_PORT"], settings=settings) + else: + _instance = chromadb.PersistentClient(path=RAG_CONFIG["CHROMA_PERSIST_DIR"]) + return _instance + + +def get_chroma() -> Any: + if _instance is None: + return init_chroma() + return _instance diff --git a/fastapi_modules/fastapi_leaudit/rag_engine/config.py b/fastapi_modules/fastapi_leaudit/rag_engine/config.py new file mode 100644 index 0000000..ff4f995 --- /dev/null +++ b/fastapi_modules/fastapi_leaudit/rag_engine/config.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +from fastapi_admin.config._settings import llm + + +def _get_str(name: str, default: str = "") -> str: + import os + return os.getenv(name, default) + + +def _get_bool(name: str, default: bool = False) -> bool: + import os + return os.getenv(name, str(default).lower()).lower() == "true" + + +def _get_int(name: str, default: int) -> int: + import os + try: + return int(os.getenv(name, str(default))) + except ValueError: + return default + + +def _get_float(name: str, default: float) -> float: + import os + try: + return float(os.getenv(name, str(default))) + except ValueError: + return default + + +RAG_CONFIG = { + "USE_SELF_HOSTED": True, + "CHROMA_PERSIST_DIR": _get_str("RAG_CHROMA_PERSIST_DIR", ".chromadb_rag"), + "CHROMA_HOST": _get_str("RAG_CHROMA_HOST", ""), + "CHROMA_PORT": _get_int("RAG_CHROMA_PORT", 8010), + "CHROMA_TOKEN": _get_str("RAG_CHROMA_TOKEN", ""), + "CHROMA_AUTH_HEADER": _get_str("RAG_CHROMA_AUTH_HEADER", "X-Chroma-Token"), + "EMBED_URL": _get_str("RAG_EMBED_URL", _get_str("GRAPH_RAG_EMBED_URL", "")), + "EMBED_KEY": _get_str("RAG_EMBED_KEY", _get_str("GRAPH_RAG_EMBED_KEY", "")), + "EMBED_MODEL": _get_str("RAG_EMBED_MODEL", _get_str("GRAPH_RAG_EMBED_MODEL", "")), + "EMBED_DIM": _get_int("RAG_EMBED_DIM", 1024), + "EMBED_BATCH_SIZE": _get_int("RAG_EMBED_BATCH_SIZE", 10), + "RERANKER_URL": _get_str("RAG_RERANKER_URL", _get_str("GRAPH_RAG_RERANKER_URL", "")), + "RERANKER_KEY": _get_str("RAG_RERANKER_KEY", _get_str("GRAPH_RAG_RERANKER_KEY", "")), + "RERANKER_MODEL": _get_str("RAG_RERANKER_MODEL", _get_str("GRAPH_RAG_RERANKER_MODEL", "")), + "LLM_BASE_URL": _get_str("LLM_BASE_URL", llm.LLM_BASE_URL), + "LLM_MODEL": _get_str("LLM_MODEL", llm.LLM_MODEL), + "LLM_API_KEY": _get_str("LLM_API_KEY", llm.LLM_API_KEY), + "VECTOR_TOP_K": _get_int("RAG_VECTOR_TOP_K", 15), + "RERANK_TOP_K": _get_int("RAG_RERANK_TOP_K", 5), + "BM25_TOP_K": _get_int("RAG_BM25_TOP_K", 15), + "RRF_K": _get_int("RAG_RRF_K", 60), + "LLM_TEMPERATURE": _get_float("RAG_LLM_TEMPERATURE", 0.3), + "LLM_MAX_TOKENS": _get_int("RAG_LLM_MAX_TOKENS", 2048), + "LLM_TIMEOUT": _get_int("RAG_LLM_TIMEOUT", 120), + "QUERY_REWRITING": _get_bool("RAG_QUERY_REWRITING", False), + "HYBRID_SEARCH": _get_bool("RAG_HYBRID_SEARCH", True), + "RERANKING": _get_bool("RAG_RERANKING", True), +} diff --git a/fastapi_modules/fastapi_leaudit/rag_engine/generator.py b/fastapi_modules/fastapi_leaudit/rag_engine/generator.py new file mode 100644 index 0000000..bf9c92f --- /dev/null +++ b/fastapi_modules/fastapi_leaudit/rag_engine/generator.py @@ -0,0 +1,144 @@ +from __future__ import annotations + +import json +import time +import uuid +from typing import AsyncGenerator + +import httpx + +from fastapi_modules.fastapi_leaudit.rag_engine.config import RAG_CONFIG + +DEFAULT_SYSTEM_PROMPT = """你是烟草行业智慧法务小助手，专注于烟草专卖法规、合同管理、行政处罚等相关法律法规。\n\n回答要求：\n- 先用一句话直接回答，再展开详细说明\n- 多个要点用编号列表\n- 关键法条和数字用 **加粗**\n- 分类信息用表格\n- 层级结构用缩进子列表\n- 不要加标题，直接输出正文""" + + +async def generate_stream( + query: str, + context_chunks: list[dict], + conversation_id: str, + message_id: str, + system_prompt: str = "", + model: str = "", + temperature: float | None = None, + max_tokens: int | None = None, + dataset_name: str = "", +) -> AsyncGenerator[str, None]: + task_id = str(uuid.uuid4()) + created_at = int(time.time()) + _model = model or RAG_CONFIG["LLM_MODEL"] + _temp = temperature if temperature is not None else RAG_CONFIG["LLM_TEMPERATURE"] + _max_tok = max_tokens or RAG_CONFIG["LLM_MAX_TOKENS"] + _prompt = system_prompt or DEFAULT_SYSTEM_PROMPT + + max_context_chars = 8000 + if context_chunks: + parts: list[str] = [] + total_len = 0 + for chunk in context_chunks: + part = f"[来源: {chunk.get('source', '未知')}]\\n{chunk.get('text', '')}" + if total_len + len(part) > max_context_chars: + break + parts.append(part) + total_len += len(part) + context_text = "\\n\\n---\\n\\n".join(parts) + user_content = f"知识库内容:\\n{context_text}\\n\\n用户问题: {query}" + else: + user_content = query + + messages = [ + {"role": "system", "content": _prompt}, + {"role": "user", "content": user_content}, + ] + + total_tokens = 0 + try: + async with httpx.AsyncClient(timeout=RAG_CONFIG["LLM_TIMEOUT"]) as client: + async with client.stream( + "POST", + f"{RAG_CONFIG['LLM_BASE_URL'].rstrip('/')}" + "/chat/completions", + json={ + "model": _model, + "messages": messages, + "temperature": _temp, + "max_tokens": _max_tok, + "stream": True, + "chat_template_kwargs": {"enable_thinking": False}, + }, + headers={ + "Content-Type": "application/json", + "Authorization": f"Bearer {RAG_CONFIG['LLM_API_KEY']}", + }, + ) as resp: + resp.raise_for_status() + async for line in resp.aiter_lines(): + if not line.startswith("data: "): + continue + payload = line[6:].strip() + if payload == "[DONE]": + break + chunk = json.loads(payload) + delta = chunk.get("choices", [{}])[0].get("delta", {}) + text = delta.get("content", "") + if text: + yield _sse_line( + { + "event": "message", + "task_id": task_id, + "message_id": message_id, + "conversation_id": conversation_id, + "answer": text, + "created_at": created_at, + } + ) + usage = chunk.get("usage") + if usage: + total_tokens = usage.get("total_tokens", total_tokens) + except Exception as exc: + yield _sse_line( + { + "event": "error", + "task_id": task_id, + "message_id": message_id, + "code": "llm_error", + "message": str(exc), + } + ) + return + + retriever_resources = [ + { + "position": i + 1, + "dataset_id": "", + "dataset_name": dataset_name, + "document_id": "", + "document_name": chunk.get("source", ""), + "data_source_type": "upload_file", + "segment_id": chunk.get("id", ""), + "retriever_from": "rag", + "score": round(chunk.get("score", 0.0), 4), + "hit_count": 0, + "word_count": len(chunk.get("text", "")), + "segment_position": i + 1, + "index_node_hash": "", + "content": chunk.get("text", "")[:500], + "page": None, + } + for i, chunk in enumerate(context_chunks) + ] + + yield _sse_line( + { + "event": "message_end", + "task_id": task_id, + "message_id": message_id, + "conversation_id": conversation_id, + "metadata": { + "usage": {"total_tokens": total_tokens}, + "retriever_resources": retriever_resources, + }, + } + ) + + +def _sse_line(data: dict) -> str: + return f"data: {json.dumps(data, ensure_ascii=False)}\\n\\n" diff --git a/fastapi_modules/fastapi_leaudit/rag_engine/question_chains.py b/fastapi_modules/fastapi_leaudit/rag_engine/question_chains.py new file mode 100644 index 0000000..1941f02 --- /dev/null +++ b/fastapi_modules/fastapi_leaudit/rag_engine/question_chains.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +import json + +import httpx + +from fastapi_modules.fastapi_leaudit.rag_engine.config import RAG_CONFIG + + +async def generate_followups(query: str, answer: str) -> list[str]: + prompt = ( + "基于用户问题和已有回答，生成 3 个适合继续追问的简短问题。" + "仅返回 JSON 数组字符串，例如 [\"问题1\", \"问题2\"]。\\n" + f"用户问题: {query}\\n回答: {answer[:1200]}" + ) + async with httpx.AsyncClient(timeout=30.0) as client: + resp = await client.post( + f"{RAG_CONFIG['LLM_BASE_URL'].rstrip('/')}" + "/chat/completions", + json={ + "model": RAG_CONFIG["LLM_MODEL"], + "messages": [{"role": "user", "content": prompt}], + "temperature": 0.5, + "max_tokens": 256, + "chat_template_kwargs": {"enable_thinking": False}, + }, + headers={ + "Content-Type": "application/json", + "Authorization": f"Bearer {RAG_CONFIG['LLM_API_KEY']}", + }, + ) + resp.raise_for_status() + content = resp.json()["choices"][0]["message"]["content"] + try: + parsed = json.loads(content) + if isinstance(parsed, list): + return [str(item).strip() for item in parsed if str(item).strip()][:3] + except Exception: + pass + return [line.strip("- 1234567890.\t") for line in content.splitlines() if line.strip()][:3] diff --git a/fastapi_modules/fastapi_leaudit/services/__init__.py b/fastapi_modules/fastapi_leaudit/services/__init__.py index d5b8ff0..6289fe8 100644 --- a/fastapi_modules/fastapi_leaudit/services/__init__.py +++ b/fastapi_modules/fastapi_leaudit/services/__init__.py @@ -14,6 +14,8 @@ from fastapi_modules.fastapi_leaudit.services.rbacAdminService import IRbacAdmin from fastapi_modules.fastapi_leaudit.services.rbacService import IRbacService from fastapi_modules.fastapi_leaudit.services.ruleConfigService import IRuleConfigService from fastapi_modules.fastapi_leaudit.services.ruleService import IRuleService +from fastapi_modules.fastapi_leaudit.services.ragDatasetService import IRagDatasetService +from fastapi_modules.fastapi_leaudit.services.ragChatService import IRagChatService __all__ = [ "IAuditService", @@ -30,4 +32,6 @@ __all__ = [ "IRbacService", "IRuleConfigService", "IRuleService", + "IRagDatasetService", + "IRagChatService", ] diff --git a/fastapi_modules/fastapi_leaudit/services/impl/documentServiceImpl.py b/fastapi_modules/fastapi_leaudit/services/impl/documentServiceImpl.py index 9183993..8d8e213 100644 --- a/fastapi_modules/fastapi_leaudit/services/impl/documentServiceImpl.py +++ b/fastapi_modules/fastapi_leaudit/services/impl/documentServiceImpl.py @@ -618,6 +618,15 @@ class DocumentServiceImpl(IDocumentService): currentUser = await self._getCurrentUserContext(CurrentUserId) documentColumns = await self._loadDocumentColumns(Session) detail = await self._getDocumentDetail(Session, DocumentId, CurrentUserId, currentUser, documentColumns) + if not detail and await self._hasCrossReviewDocumentAccess(Session, DocumentId, CurrentUserId): + detail = await self._getDocumentDetail( + Session, + DocumentId, + CurrentUserId, + currentUser, + documentColumns, + BypassScopeCheck=True, + ) if not detail: raise LeauditException(StatusCodeEnum.HTTP_404_NOT_FOUND, "文档不存在或无权访问") @@ -679,6 +688,15 @@ class DocumentServiceImpl(IDocumentService): currentUser = await self._getCurrentUserContext(CurrentUserId) documentColumns = await self._loadDocumentColumns(Session) detail = await self._getDocumentDetail(Session, documentId, CurrentUserId, currentUser, documentColumns) + if not detail and await self._hasCrossReviewDocumentAccess(Session, documentId, CurrentUserId): + detail = await self._getDocumentDetail( + Session, + documentId, + CurrentUserId, + currentUser, + documentColumns, + BypassScopeCheck=True, + ) if not detail: raise LeauditException(StatusCodeEnum.HTTP_404_NOT_FOUND, "文档不存在或无权访问") @@ -742,6 +760,15 @@ class DocumentServiceImpl(IDocumentService): currentUser = await self._getCurrentUserContext(CurrentUserId) documentColumns = await self._loadDocumentColumns(Session) detail = await self._getDocumentDetail(Session, DocumentId, CurrentUserId, currentUser, documentColumns) + if not detail and await self._hasCrossReviewDocumentAccess(Session, DocumentId, CurrentUserId): + detail = await self._getDocumentDetail( + Session, + DocumentId, + CurrentUserId, + currentUser, + documentColumns, + BypassScopeCheck=True, + ) if not detail: raise LeauditException(StatusCodeEnum.HTTP_404_NOT_FOUND, "文档不存在或无权访问") @@ -1601,19 +1628,21 @@ class DocumentServiceImpl(IDocumentService): CurrentUserId: int, CurrentUser: dict[str, Any], DocumentColumns: set[str], + BypassScopeCheck: bool = False, ) -> DocumentDetailVO | None: """查询单文档详情，并附带历史版本。""" params: dict[str, object] = {"id": DocumentId} filters = ["d.id = :id", "d.deleted_at IS NULL", "f.is_active = true", "f.file_role = 'primary'"] - filters.extend( - self._buildDocumentScopeFilters( - CurrentUserId=CurrentUserId, - CurrentUser=CurrentUser, - Params=params, - DocumentAlias="d", - FileAlias="f", + if not BypassScopeCheck: + filters.extend( + self._buildDocumentScopeFilters( + CurrentUserId=CurrentUserId, + CurrentUser=CurrentUser, + Params=params, + DocumentAlias="d", + FileAlias="f", + ) ) - ) whereClause = " AND ".join(filters) groupIdSelectExpr = "d.group_id" if "group_id" in DocumentColumns else "NULL::bigint" @@ -1832,6 +1861,38 @@ class DocumentServiceImpl(IDocumentService): attachments=attachments, ) + async def _hasCrossReviewDocumentAccess(self, Session, DocumentId: int, CurrentUserId: int) -> bool: + """判断当前用户是否作为交叉评查任务成员拥有文档访问权。""" + if not await self._tableExists(Session, "leaudit_cross_review_task_documents"): + return False + if not await self._tableExists(Session, "leaudit_cross_review_task_members"): + return False + if not await self._tableExists(Session, "leaudit_cross_review_tasks"): + return False + + row = ( + await Session.execute( + text( + """ + SELECT 1 + FROM leaudit_cross_review_task_documents td + JOIN leaudit_cross_review_task_members tm + ON tm.task_id = td.task_id + JOIN leaudit_cross_review_tasks t + ON t.id = td.task_id + WHERE td.document_id = :document_id + AND tm.user_id = :user_id + AND td.delete_time IS NULL + AND tm.delete_time IS NULL + AND t.delete_time IS NULL + LIMIT 1 + """ + ), + {"document_id": DocumentId, "user_id": CurrentUserId}, + ) + ).first() + return bool(row) + def _buildDocumentScopeFilters( self, CurrentUserId: int, diff --git a/fastapi_modules/fastapi_leaudit/services/impl/ragChatServiceImpl.py b/fastapi_modules/fastapi_leaudit/services/impl/ragChatServiceImpl.py new file mode 100644 index 0000000..0e3c417 --- /dev/null +++ b/fastapi_modules/fastapi_leaudit/services/impl/ragChatServiceImpl.py @@ -0,0 +1,589 @@ +from __future__ import annotations + +import json +import uuid +from typing import AsyncGenerator + +from sqlalchemy import text + +from fastapi_common.fastapi_common_sqlalchemy.database import GetAsyncSession +from fastapi_common.fastapi_common_web.domain.responses import StatusCodeEnum +from fastapi_common.fastapi_common_web.exception.LeauditException import LeauditException + +from fastapi_modules.fastapi_leaudit.domian.Dto.ragChatDto import ( + RagConversationRenameDTO, + RagMessageFeedbackDTO, +) +from fastapi_modules.fastapi_leaudit.domian.vo.ragChatVo import ( + RagAppParametersVO, + RagChatAppListVO, + RagChatAppVO, + RagConversationItemVO, + RagConversationPageVO, + RagConversationRenameVO, + RagMessageItemVO, + RagMessagePageVO, + RagOperationResultVO, +) +from fastapi_modules.fastapi_leaudit.rag_engine.generator import generate_stream +from fastapi_modules.fastapi_leaudit.rag_engine.question_chains import generate_followups +from fastapi_modules.fastapi_leaudit.services.ragChatService import IRagChatService + + +class RagChatServiceImpl(IRagChatService): + async def GetApps(self, CurrentUserId: int, UserArea: str | None, UserRole: str | None) -> RagChatAppListVO: + apps = await self._load_apps(UserArea, UserRole, only_default=False) + return RagChatAppListVO(data=apps, total=len(apps)) + + async def GetDefaultApp(self, CurrentUserId: int, UserArea: str | None, UserRole: str | None) -> RagChatAppVO | None: + apps = await self._load_apps(UserArea, UserRole, only_default=True) + if apps: + return apps[0] + all_apps = await self._load_apps(UserArea, UserRole, only_default=False) + return all_apps[0] if all_apps else None + + async def SendMessage( + self, + CurrentUserId: int, + UserName: str, + UserArea: str | None, + UserRole: str | None, + Query: str, + ConversationId: str | None, + AppId: int | None, + ) -> AsyncGenerator[bytes, None]: + if not Query.strip(): + raise LeauditException(StatusCodeEnum.HTTP_400_BAD_REQUEST, "问题不能为空") + + app = await self._resolve_app(AppId, UserArea, UserRole) + if not app: + raise LeauditException(StatusCodeEnum.HTTP_404_NOT_FOUND, "未配置可用聊天应用") + + conversationId = await self._ensure_conversation(CurrentUserId, ConversationId, app["id"]) + messageId = str(uuid.uuid4()) + + async with GetAsyncSession() as session: + async with session.begin(): + await session.execute( + text( + """ + INSERT INTO rag_message (message_id, conversation_id, role, content, sources, metadata) + VALUES (:message_id, :conversation_id, 'user', :content, '[]'::jsonb, '{}'::jsonb) + """ + ), + { + "message_id": str(uuid.uuid4()), + "conversation_id": conversationId, + "content": Query, + }, + ) + + context_chunks, dataset_name = await self._retrieve_context(app.get("dataset_id"), Query) + collected_answer = "" + held_message_end: bytes | None = None + + async for chunk in generate_stream( + query=Query, + context_chunks=context_chunks, + conversation_id=conversationId, + message_id=messageId, + system_prompt=app.get("system_prompt") or "", + model=app.get("llm_model") or "", + temperature=app.get("temperature"), + max_tokens=app.get("max_tokens"), + dataset_name=dataset_name, + ): + chunk_bytes = chunk.encode("utf-8") + for line in chunk.strip().split("\n"): + if not line.startswith("data: "): + continue + data = json.loads(line[6:]) + if data.get("event") == "message": + collected_answer += data.get("answer", "") + elif data.get("event") == "message_end": + held_message_end = chunk_bytes + continue + if held_message_end is None: + yield chunk_bytes + + followups: list[str] = [] + try: + followups = await generate_followups(Query, collected_answer) + except Exception: + followups = [] + + if held_message_end: + try: + for line in held_message_end.decode("utf-8").strip().split("\n"): + if not line.startswith("data: "): + continue + end_data = json.loads(line[6:]) + if end_data.get("event") == "message_end": + end_data.setdefault("metadata", {})["suggested_questions"] = followups + yield f"data: {json.dumps(end_data, ensure_ascii=False)}\\n\\n".encode("utf-8") + except Exception: + yield held_message_end + + async with GetAsyncSession() as session: + async with session.begin(): + await session.execute( + text( + """ + INSERT INTO rag_message (message_id, conversation_id, role, content, sources, metadata) + VALUES (:message_id, :conversation_id, 'assistant', :content, CAST(:sources AS jsonb), CAST(:metadata AS jsonb)) + """ + ), + { + "message_id": messageId, + "conversation_id": conversationId, + "content": collected_answer, + "sources": json.dumps(self._build_sources(context_chunks, dataset_name), ensure_ascii=False), + "metadata": json.dumps({"suggested_questions": followups}, ensure_ascii=False), + }, + ) + await session.execute( + text( + "UPDATE rag_conversation SET updated_at = NOW() WHERE conversation_id = :conversation_id" + ), + {"conversation_id": conversationId}, + ) + + async def GetConversations(self, CurrentUserId: int, AppId: int | None, Page: int, PageSize: int) -> RagConversationPageVO: + async with GetAsyncSession() as session: + rows = ( + await session.execute( + text( + """ + SELECT conversation_id, name, introduction, created_at, updated_at + FROM rag_conversation + WHERE user_id = :user_id + AND deleted_at IS NULL + AND (:app_id IS NULL OR app_id = :app_id) + ORDER BY updated_at DESC + OFFSET :offset LIMIT :limit + """ + ), + { + "user_id": CurrentUserId, + "app_id": AppId, + "offset": max(Page - 1, 0) * PageSize, + "limit": PageSize + 1, + }, + ) + ).mappings().all() + has_more = len(rows) > PageSize + items = rows[:PageSize] + return RagConversationPageVO( + data=[ + RagConversationItemVO( + id=row["conversation_id"], + name=row["name"], + introduction=row.get("introduction") or "", + createdAt=int(row["created_at"].timestamp()) if row.get("created_at") else 0, + updatedAt=int(row["updated_at"].timestamp()) if row.get("updated_at") else 0, + ) + for row in items + ], + hasMore=has_more, + limit=PageSize, + ) + + async def GetConversationMessages(self, CurrentUserId: int, ConversationId: str, Page: int, PageSize: int) -> RagMessagePageVO: + await self._ensure_conversation_owner(CurrentUserId, ConversationId) + async with GetAsyncSession() as session: + rows = ( + await session.execute( + text( + """ + SELECT message_id, role, content, sources, feedback, created_at + FROM rag_message + WHERE conversation_id = :conversation_id + ORDER BY created_at ASC + OFFSET :offset LIMIT :limit + """ + ), + { + "conversation_id": ConversationId, + "offset": max(Page - 1, 0) * PageSize, + "limit": PageSize + 1, + }, + ) + ).mappings().all() + has_more = len(rows) > PageSize + items = rows[:PageSize] + data: list[RagMessageItemVO] = [] + idx = 0 + while idx < len(items): + row = items[idx] + if row["role"] == "user": + answer = items[idx + 1] if idx + 1 < len(items) and items[idx + 1]["role"] == "assistant" else None + data.append( + RagMessageItemVO( + id=(answer["message_id"] if answer else row["message_id"]), + conversationId=ConversationId, + query=row["content"], + answer=answer["content"] if answer else "", + feedback=({"rating": answer["feedback"]} if answer and answer.get("feedback") else None), + retrieverResources=(answer.get("sources") if answer else None), + createdAt=int(row["created_at"].timestamp()) if row.get("created_at") else 0, + ) + ) + idx += 2 if answer else 1 + else: + idx += 1 + return RagMessagePageVO(data=data, hasMore=has_more, limit=PageSize) + + async def RenameConversation(self, CurrentUserId: int, ConversationId: str, Body: RagConversationRenameDTO) -> RagConversationRenameVO: + await self._ensure_conversation_owner(CurrentUserId, ConversationId) + async with GetAsyncSession() as session: + async with session.begin(): + await session.execute( + text( + "UPDATE rag_conversation SET name = :name, updated_at = NOW() WHERE conversation_id = :conversation_id" + ), + {"name": Body.name, "conversation_id": ConversationId}, + ) + return RagConversationRenameVO(result="success", name=Body.name) + + async def DeleteConversation(self, CurrentUserId: int, ConversationId: str) -> RagOperationResultVO: + await self._ensure_conversation_owner(CurrentUserId, ConversationId) + async with GetAsyncSession() as session: + async with session.begin(): + await session.execute( + text( + "UPDATE rag_conversation SET deleted_at = NOW(), updated_at = NOW() WHERE conversation_id = :conversation_id" + ), + {"conversation_id": ConversationId}, + ) + return RagOperationResultVO(result="success") + + async def UpdateFeedback(self, CurrentUserId: int, MessageId: str, Body: RagMessageFeedbackDTO) -> RagOperationResultVO: + async with GetAsyncSession() as session: + owner = ( + await session.execute( + text( + """ + SELECT c.user_id + FROM rag_message m + JOIN rag_conversation c ON c.conversation_id = m.conversation_id + WHERE m.message_id = :message_id AND c.deleted_at IS NULL + LIMIT 1 + """ + ), + {"message_id": MessageId}, + ) + ).scalar_one_or_none() + if owner is None: + raise LeauditException(StatusCodeEnum.HTTP_404_NOT_FOUND, "消息不存在") + if int(owner) != CurrentUserId: + raise LeauditException(StatusCodeEnum.HTTP_403_FORBIDDEN, "当前用户无权修改该消息反馈") + async with session.begin(): + await session.execute( + text("UPDATE rag_message SET feedback = :feedback WHERE message_id = :message_id"), + {"feedback": Body.rating, "message_id": MessageId}, + ) + return RagOperationResultVO(result="success") + + async def GetAppParameters( + self, + CurrentUserId: int, + UserArea: str | None, + UserRole: str | None, + AppId: int | None, + ) -> RagAppParametersVO: + app = await self._resolve_app(AppId, UserArea, UserRole) + if not app: + return RagAppParametersVO() + try: + suggested = json.loads(app.get("suggested_questions") or "[]") + if not isinstance(suggested, list): + suggested = [] + except Exception: + suggested = [] + return RagAppParametersVO( + openingStatement=app.get("opening_statement") or "", + suggestedQuestions=[str(item) for item in suggested[:6]], + userInputForm=[], + fileUpload={"image": {"enabled": False}}, + ) + + async def _load_apps(self, user_area: str | None, user_role: str | None, only_default: bool) -> list[RagChatAppVO]: + async with GetAsyncSession() as session: + sql = ( + """ + SELECT a.id, a.name, a.description, a.is_default + FROM rag_chat_app a + LEFT JOIN rag_dataset d ON d.id = a.dataset_id AND d.deleted_at IS NULL + WHERE a.deleted_at IS NULL + AND a.status = 1 + AND (:only_default = FALSE OR a.is_default = TRUE) + AND ( + :is_provincial = TRUE + OR a.area IN (:user_area, '省级', '') + OR COALESCE(d.is_public, FALSE) = TRUE + ) + ORDER BY a.sort_order ASC, a.created_at DESC + """ + ) + rows = ( + await session.execute( + text(sql), + { + "only_default": only_default, + "is_provincial": user_role == "provincial_admin", + "user_area": user_area or "", + }, + ) + ).mappings().all() + return [ + RagChatAppVO( + appId=str(row["id"]), + appName=row["name"], + description=row.get("description") or "", + isDefault=bool(row.get("is_default")), + ) + for row in rows + ] + + async def _resolve_app(self, app_id: int | None, user_area: str | None, user_role: str | None) -> dict | None: + async with GetAsyncSession() as session: + params = { + "app_id": app_id, + "user_area": user_area or "", + "is_provincial": user_role == "provincial_admin", + } + base_sql = ( + """ + SELECT a.id, a.name, a.description, a.area, a.dataset_id, a.system_prompt, + a.llm_model, a.temperature, a.max_tokens, a.opening_statement, + a.suggested_questions, a.is_default, COALESCE(d.is_public, FALSE) AS dataset_public, + COALESCE(d.name, '') AS dataset_name + FROM rag_chat_app a + LEFT JOIN rag_dataset d ON d.id = a.dataset_id AND d.deleted_at IS NULL + WHERE a.deleted_at IS NULL AND a.status = 1 + """ + ) + if app_id is not None: + row = ( + await session.execute( + text(base_sql + " AND a.id = :app_id LIMIT 1"), + params, + ) + ).mappings().first() + if row and self._app_visible(row, user_area, user_role): + return dict(row) + row = ( + await session.execute( + text(base_sql + " AND a.is_default = TRUE ORDER BY a.sort_order ASC, a.created_at DESC LIMIT 1"), + params, + ) + ).mappings().first() + if row and self._app_visible(row, user_area, user_role): + return dict(row) + row = ( + await session.execute( + text(base_sql + " ORDER BY a.sort_order ASC, a.created_at DESC LIMIT 1"), + params, + ) + ).mappings().first() + return dict(row) if row and self._app_visible(row, user_area, user_role) else None + + def _app_visible(self, row: dict, user_area: str | None, user_role: str | None) -> bool: + if user_role == "provincial_admin": + return True + area = row.get("area") or "" + return area in ("", "省级", user_area or "") or bool(row.get("dataset_public")) + + async def _ensure_conversation(self, user_id: int, conversation_id: str | None, app_id: int | None) -> str: + if conversation_id and conversation_id != "-1": + async with GetAsyncSession() as session: + row = ( + await session.execute( + text( + """ + SELECT conversation_id, user_id + FROM rag_conversation + WHERE conversation_id = :conversation_id + AND deleted_at IS NULL + LIMIT 1 + """ + ), + {"conversation_id": conversation_id}, + ) + ).mappings().first() + if row: + if int(row["user_id"]) != user_id: + raise LeauditException(StatusCodeEnum.HTTP_403_FORBIDDEN, "当前用户无权使用该会话") + return str(row["conversation_id"]) + conversation_id = str(uuid.uuid4()) + async with GetAsyncSession() as session: + async with session.begin(): + await session.execute( + text( + """ + INSERT INTO rag_conversation (conversation_id, user_id, app_id, name, introduction) + VALUES (:conversation_id, :user_id, :app_id, '新对话', '') + """ + ), + {"conversation_id": conversation_id, "user_id": user_id, "app_id": app_id}, + ) + return conversation_id + + async def _ensure_conversation_owner(self, user_id: int, conversation_id: str) -> None: + async with GetAsyncSession() as session: + owner = ( + await session.execute( + text( + "SELECT user_id FROM rag_conversation WHERE conversation_id = :conversation_id AND deleted_at IS NULL LIMIT 1" + ), + {"conversation_id": conversation_id}, + ) + ).scalar_one_or_none() + if owner is None: + raise LeauditException(StatusCodeEnum.HTTP_404_NOT_FOUND, "会话不存在") + if int(owner) != user_id: + raise LeauditException(StatusCodeEnum.HTTP_403_FORBIDDEN, "当前用户无权访问该会话") + + async def _retrieve_context(self, dataset_id: int | None, query: str) -> tuple[list[dict], str]: + if not dataset_id: + return [], "" + async with GetAsyncSession() as session: + dataset = ( + await session.execute( + text( + """ + SELECT id, name, collection_name, retrieval_model + FROM rag_dataset + WHERE id = :dataset_id AND deleted_at IS NULL + LIMIT 1 + """ + ), + {"dataset_id": dataset_id}, + ) + ).mappings().first() + if not dataset: + return [], "" + retrieval_model = dataset.get("retrieval_model") or {} + top_k = int(retrieval_model.get("top_k") or 5) + score_threshold = None + if retrieval_model.get("score_threshold_enabled"): + try: + score_threshold = float(retrieval_model.get("score_threshold")) + except (TypeError, ValueError): + score_threshold = None + try: + from fastapi_modules.fastapi_leaudit.rag_engine.chroma_client import get_chroma + except Exception: + return [], dataset.get("name") or "" + try: + collection = get_chroma().get_or_create_collection(dataset["collection_name"]) + result = collection.query(query_texts=[query], n_results=max(top_k, 1)) + docs = (result.get("documents") or [[]])[0] + metas = (result.get("metadatas") or [[]])[0] + distances = (result.get("distances") or [[]])[0] + chunks: list[dict] = [] + for idx, doc in enumerate(docs): + meta = metas[idx] if idx < len(metas) else {} + dist = distances[idx] if idx < len(distances) else 0.0 + score = 1 - float(dist or 0.0) + if score_threshold is not None and score < score_threshold: + continue + chunks.append( + { + "id": str(meta.get("id") or idx), + "text": doc, + "source": meta.get("source") or meta.get("document_name") or dataset.get("name") or "", + "score": score, + "chunk_index": idx, + "document_name": meta.get("document_name") or meta.get("source") or "", + } + ) + chunks = await self._hydrate_document_hits(dataset_id, chunks) + return chunks[:top_k], dataset.get("name") or "" + except Exception: + return [], dataset.get("name") or "" + + def _build_sources(self, context_chunks: list[dict], dataset_name: str) -> list[dict]: + return [ + { + "position": index + 1, + "dataset_id": str(chunk.get("dataset_id") or ""), + "dataset_name": dataset_name, + "document_id": str(chunk.get("document_id") or ""), + "document_name": chunk.get("document_name") or chunk.get("source", ""), + "data_source_type": "upload_file", + "segment_id": chunk.get("id", ""), + "retriever_from": "rag", + "score": round(chunk.get("score", 0.0), 4), + "hit_count": chunk.get("hit_count", 0), + "word_count": len(chunk.get("text", "")), + "segment_position": index + 1, + "index_node_hash": "", + "content": chunk.get("text", "")[:500], + "page": None, + } + for index, chunk in enumerate(context_chunks) + ] + + async def _hydrate_document_hits(self, dataset_id: int, chunks: list[dict]) -> list[dict]: + source_names = sorted( + { + str(chunk.get("document_name") or chunk.get("source") or "").strip() + for chunk in chunks + if str(chunk.get("document_name") or chunk.get("source") or "").strip() + } + ) + if not source_names: + return chunks + + async with GetAsyncSession() as session: + rows = ( + await session.execute( + text( + """ + SELECT id, original_name, enabled, hit_count + FROM rag_document + WHERE dataset_id = :dataset_id + AND deleted_at IS NULL + AND original_name = ANY(:source_names) + """ + ), + { + "dataset_id": dataset_id, + "source_names": source_names, + }, + ) + ).mappings().all() + + document_map = {str(row["original_name"]): row for row in rows} + visible_chunks: list[dict] = [] + hit_document_ids: list[int] = [] + for chunk in chunks: + source_name = str(chunk.get("document_name") or chunk.get("source") or "").strip() + document = document_map.get(source_name) + if document and not bool(document.get("enabled")): + continue + if document: + chunk["document_id"] = document["id"] + chunk["dataset_id"] = dataset_id + chunk["document_name"] = document["original_name"] + chunk["hit_count"] = document.get("hit_count") or 0 + hit_document_ids.append(int(document["id"])) + visible_chunks.append(chunk) + + if hit_document_ids: + async with GetAsyncSession() as session: + async with session.begin(): + await session.execute( + text( + """ + UPDATE rag_document + SET hit_count = hit_count + 1, + updated_at = NOW() + WHERE id = ANY(:document_ids) + """ + ), + {"document_ids": sorted(set(hit_document_ids))}, + ) + + return visible_chunks diff --git a/fastapi_modules/fastapi_leaudit/services/impl/ragDatasetServiceImpl.py b/fastapi_modules/fastapi_leaudit/services/impl/ragDatasetServiceImpl.py new file mode 100644 index 0000000..a7b40f2 --- /dev/null +++ b/fastapi_modules/fastapi_leaudit/services/impl/ragDatasetServiceImpl.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +from sqlalchemy import text + +from fastapi_common.fastapi_common_sqlalchemy.database import GetAsyncSession + +from fastapi_modules.fastapi_leaudit.domian.vo.ragDatasetVo import RagDatasetItemVO, RagDatasetPageVO +from fastapi_modules.fastapi_leaudit.services.ragDatasetService import IRagDatasetService + + +class RagDatasetServiceImpl(IRagDatasetService): + async def GetMyDatasets(self, CurrentUserId: int, UserArea: str | None, UserRole: str | None) -> RagDatasetPageVO: + async with GetAsyncSession() as session: + rows = ( + await session.execute( + text( + """ + SELECT id, name, description, area, is_public, is_default, document_count, total_chunks, status + FROM rag_dataset + WHERE deleted_at IS NULL + AND status = 1 + AND ( + :is_provincial = TRUE + OR area IN (:user_area, '省级', '') + OR is_public = TRUE + ) + ORDER BY sort_order ASC, created_at DESC + """ + ), + { + "is_provincial": UserRole == "provincial_admin", + "user_area": UserArea or "", + }, + ) + ).mappings().all() + return RagDatasetPageVO( + data=[ + RagDatasetItemVO( + id=row["id"], + name=row["name"], + description=row.get("description") or "", + area=row.get("area") or "", + isPublic=bool(row.get("is_public")), + isDefault=bool(row.get("is_default")), + documentCount=row.get("document_count") or 0, + totalChunks=row.get("total_chunks") or 0, + status=row.get("status") or 1, + ) + for row in rows + ], + total=len(rows), + ) diff --git a/fastapi_modules/fastapi_leaudit/services/impl/rbacAdminServiceImpl.py b/fastapi_modules/fastapi_leaudit/services/impl/rbacAdminServiceImpl.py index a81dc67..7734fcf 100644 --- a/fastapi_modules/fastapi_leaudit/services/impl/rbacAdminServiceImpl.py +++ b/fastapi_modules/fastapi_leaudit/services/impl/rbacAdminServiceImpl.py @@ -223,6 +223,13 @@ class RbacAdminServiceImpl(IRbacAdminService): {"permission_key": "rbac:user_roles:write", "display_name": "分配用户角色", "module": "rbac", "resource": "user_roles", "action": "write", "api_method": "POST", "api_path": "/api/v3/rbac/users/{user_id}/roles", "route_path": "/role-permissions"}, {"permission_key": "rbac:role_routes:write", "display_name": "配置角色菜单", "module": "rbac", "resource": "role_routes", "action": "write", "api_method": "PUT", "api_path": "/api/rbac/roles/{role_id}/routes", "route_path": "/role-permissions"}, {"permission_key": "rbac:role_permissions:write", "display_name": "配置角色权限", "module": "rbac", "resource": "role_permissions", "action": "write", "api_method": "POST", "api_path": "/api/v3/rbac/role-permissions", "route_path": "/role-permissions"}, + {"permission_key": "rag:app:read", "display_name": "查看 RAG 应用", "module": "rag", "resource": "app", "action": "read", "api_method": "GET", "api_path": "/api/v3/rag/apps", "route_path": "/chat-with-llm"}, + {"permission_key": "rag:chat:use", "display_name": "使用 RAG 对话", "module": "rag", "resource": "chat", "action": "use", "api_method": "POST", "api_path": "/api/v3/rag/chat/messages", "route_path": "/chat-with-llm"}, + {"permission_key": "rag:conversation:read", "display_name": "查看 RAG 会话", "module": "rag", "resource": "conversation", "action": "read", "api_method": "GET", "api_path": "/api/v3/rag/chat/conversations", "route_path": "/chat-with-llm"}, + {"permission_key": "rag:conversation:update", "display_name": "重命名 RAG 会话", "module": "rag", "resource": "conversation", "action": "update", "api_method": "PATCH", "api_path": "/api/v3/rag/chat/conversations/{ConversationId}", "route_path": "/chat-with-llm"}, + {"permission_key": "rag:conversation:delete", "display_name": "删除 RAG 会话", "module": "rag", "resource": "conversation", "action": "delete", "api_method": "DELETE", "api_path": "/api/v3/rag/chat/conversations/{ConversationId}", "route_path": "/chat-with-llm"}, + {"permission_key": "rag:message:feedback", "display_name": "反馈 RAG 消息", "module": "rag", "resource": "message", "action": "feedback", "api_method": "POST", "api_path": "/api/v3/rag/chat/messages/{MessageId}/feedback", "route_path": "/chat-with-llm"}, + {"permission_key": "rag:dataset:read", "display_name": "查看 RAG 知识库", "module": "rag", "resource": "dataset", "action": "read", "api_method": "GET", "api_path": "/api/v3/rag/datasets/my", "route_path": "/chat-with-llm"}, ] async def ListRoles(self, CurrentUserId: int, Page: int, PageSize: int, RoleKey: str | None, RoleName: str | None, IncludeSystem: bool) -> RoleListVO: diff --git a/fastapi_modules/fastapi_leaudit/services/ragChatService.py b/fastapi_modules/fastapi_leaudit/services/ragChatService.py new file mode 100644 index 0000000..b943605 --- /dev/null +++ b/fastapi_modules/fastapi_leaudit/services/ragChatService.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +from abc import ABC, abstractmethod +from typing import AsyncGenerator + +from fastapi_modules.fastapi_leaudit.domian.Dto.ragChatDto import ( + RagConversationRenameDTO, + RagMessageFeedbackDTO, +) +from fastapi_modules.fastapi_leaudit.domian.vo.ragChatVo import ( + RagAppParametersVO, + RagChatAppListVO, + RagChatAppVO, + RagConversationPageVO, + RagConversationRenameVO, + RagMessagePageVO, + RagOperationResultVO, +) + + +class IRagChatService(ABC): + @abstractmethod + async def GetApps(self, CurrentUserId: int, UserArea: str | None, UserRole: str | None) -> RagChatAppListVO: ... + + @abstractmethod + async def GetDefaultApp(self, CurrentUserId: int, UserArea: str | None, UserRole: str | None) -> RagChatAppVO | None: ... + + @abstractmethod + async def SendMessage( + self, + CurrentUserId: int, + UserName: str, + UserArea: str | None, + UserRole: str | None, + Query: str, + ConversationId: str | None, + AppId: int | None, + ) -> AsyncGenerator[bytes, None]: ... + + @abstractmethod + async def GetConversations(self, CurrentUserId: int, AppId: int | None, Page: int, PageSize: int) -> RagConversationPageVO: ... + + @abstractmethod + async def GetConversationMessages(self, CurrentUserId: int, ConversationId: str, Page: int, PageSize: int) -> RagMessagePageVO: ... + + @abstractmethod + async def RenameConversation(self, CurrentUserId: int, ConversationId: str, Body: RagConversationRenameDTO) -> RagConversationRenameVO: ... + + @abstractmethod + async def DeleteConversation(self, CurrentUserId: int, ConversationId: str) -> RagOperationResultVO: ... + + @abstractmethod + async def UpdateFeedback(self, CurrentUserId: int, MessageId: str, Body: RagMessageFeedbackDTO) -> RagOperationResultVO: ... + + @abstractmethod + async def GetAppParameters( + self, + CurrentUserId: int, + UserArea: str | None, + UserRole: str | None, + AppId: int | None, + ) -> RagAppParametersVO: ... diff --git a/fastapi_modules/fastapi_leaudit/services/ragDatasetService.py b/fastapi_modules/fastapi_leaudit/services/ragDatasetService.py new file mode 100644 index 0000000..f71048d --- /dev/null +++ b/fastapi_modules/fastapi_leaudit/services/ragDatasetService.py @@ -0,0 +1,10 @@ +from __future__ import annotations + +from abc import ABC, abstractmethod + +from fastapi_modules.fastapi_leaudit.domian.vo.ragDatasetVo import RagDatasetPageVO + + +class IRagDatasetService(ABC): + @abstractmethod + async def GetMyDatasets(self, CurrentUserId: int, UserArea: str | None, UserRole: str | None) -> RagDatasetPageVO: ... diff --git a/pyproject.toml b/pyproject.toml index e1ab208..1c79224 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,6 +14,7 @@ dependencies = [ "pydantic>=2.10.0", "pydantic-settings>=2.7.0", "httpx>=0.28.0", + "chromadb>=0.5.23", "celery>=5.4.0", "redis>=5.2.0", "tomli>=2.2.0", diff --git a/scripts/merge_document_version_groups.py b/scripts/merge_document_version_groups.py new file mode 100644 index 0000000..0e0f440 --- /dev/null +++ b/scripts/merge_document_version_groups.py @@ -0,0 +1,277 @@ +#!/usr/bin/env python3 +"""Merge historical split document version chains under the new root-group rule. + +Default mode is dry-run. Use --apply to write changes. + +Merge key: +- region +- root business group (一级分组) +- normalized_name + +Within one merge group: +- sort by created_at ASC, then document_id ASC +- keep the earliest document as root version +- reuse the earliest document's version_group_key as the canonical chain key +- renumber version_no from 1..N +- link previous_version_id sequentially +- mark only the newest document as is_latest_version = true +""" + +from __future__ import annotations + +import argparse +import asyncio +from collections import defaultdict +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import asyncpg + + +ROOT = Path(__file__).resolve().parents[1] +APP_TOML = ROOT / "app.toml" + + +@dataclass +class CandidateRow: + document_id: int + region: str + normalized_name: str + root_group_id: int | None + root_group_name: str | None + type_id: int | None + type_name: str | None + group_id: int | None + group_name: str | None + version_group_key: str + version_no: int + is_latest_version: bool + file_name: str | None + updated_at: Any + created_at: Any + + +def load_target_dsn() -> str: + try: + import tomllib + except ImportError: # pragma: no cover + import tomli as tomllib + + with APP_TOML.open("rb") as fh: + config = tomllib.load(fh) + db = config["DB"] + return ( + f"postgresql://{db['USER']}:{db['PASSWORD']}" + f"@{db['HOST']}:{db['PORT']}/{db['NAME']}" + ) + + +async def fetch_candidates( + conn: asyncpg.Connection, + *, + region: str | None, + normalized_name: str | None, +) -> list[CandidateRow]: + filters = ["d.deleted_at IS NULL"] + params: list[Any] = [] + + if region: + params.append(region) + filters.append(f"d.region = ${len(params)}") + if normalized_name: + params.append(normalized_name.strip().lower()) + filters.append(f"d.normalized_name = ${len(params)}") + + where_clause = " AND ".join(filters) + + rows = await conn.fetch( + f""" + WITH doc_scope AS ( + SELECT + d.id AS document_id, + d.region, + d.normalized_name, + d.version_group_key, + d.version_no, + d.is_latest_version, + d.type_id, + dt.name AS type_name, + d.group_id, + child.name AS group_name, + COALESCE( + CASE + WHEN child.id IS NULL THEN NULL + WHEN COALESCE(child.pid, 0) = 0 THEN child.id + ELSE child.pid + END, + inferred.root_group_id + ) AS root_group_id, + COALESCE(root.name, inferred.root_group_name) AS root_group_name, + d.created_at, + d.updated_at, + f.file_name + FROM leaudit_documents d + LEFT JOIN leaudit_document_types dt + ON dt.id = d.type_id + LEFT JOIN leaudit_evaluation_point_groups child + ON child.id = d.group_id + LEFT JOIN leaudit_evaluation_point_groups root + ON root.id = CASE + WHEN child.id IS NULL THEN NULL + WHEN COALESCE(child.pid, 0) = 0 THEN child.id + ELSE child.pid + END + LEFT JOIN ( + SELECT + child2.document_type_id, + CASE WHEN COUNT(DISTINCT child2.pid) = 1 THEN MIN(child2.pid) END AS root_group_id, + CASE WHEN COUNT(DISTINCT child2.pid) = 1 THEN MIN(parent.name) END AS root_group_name + FROM leaudit_evaluation_point_groups child2 + JOIN leaudit_evaluation_point_groups parent + ON parent.id = child2.pid + WHERE COALESCE(child2.pid, 0) <> 0 + AND child2.deleted_at IS NULL + AND child2.is_enabled = true + AND child2.document_type_id IS NOT NULL + GROUP BY child2.document_type_id + ) inferred + ON inferred.document_type_id = d.type_id + LEFT JOIN leaudit_document_files f + ON f.document_id = d.id + AND f.is_active = true + AND f.file_role = 'primary' + WHERE {where_clause} + ), + conflict_keys AS ( + SELECT + region, + normalized_name, + root_group_id + FROM doc_scope + WHERE normalized_name IS NOT NULL + AND normalized_name <> '' + AND root_group_id IS NOT NULL + GROUP BY region, normalized_name, root_group_id + HAVING COUNT(DISTINCT version_group_key) > 1 + ) + SELECT ds.* + FROM doc_scope ds + JOIN conflict_keys ck + ON ck.region = ds.region + AND ck.normalized_name = ds.normalized_name + AND ck.root_group_id = ds.root_group_id + ORDER BY ds.region, ds.root_group_name, ds.normalized_name, ds.created_at ASC, ds.document_id ASC + """, + *params, + ) + + return [CandidateRow(**dict(row)) for row in rows] + + +def group_candidates(rows: list[CandidateRow]) -> list[list[CandidateRow]]: + grouped: dict[tuple[str, str, int], list[CandidateRow]] = defaultdict(list) + for row in rows: + if row.root_group_id is None: + continue + grouped[(row.region, row.normalized_name, row.root_group_id)].append(row) + + result: list[list[CandidateRow]] = [] + for group_rows in grouped.values(): + group_rows.sort(key=lambda item: (item.created_at, item.document_id)) + result.append(group_rows) + result.sort(key=lambda items: (items[0].region, items[0].root_group_name or "", items[0].normalized_name)) + return result + + +def print_plan(groups: list[list[CandidateRow]], limit: int) -> None: + print(f"计划处理候选组: {len(groups)}") + if not groups: + return + + for index, rows in enumerate(groups[:limit], start=1): + first = rows[0] + canonical_key = first.version_group_key + root_id = first.document_id + latest_id = rows[-1].document_id + print() + print( + f"[{index}] 地区={first.region} | 一级分组={first.root_group_name or first.root_group_id} | " + f"名称={first.normalized_name} | 目标链={canonical_key} | 根文档={root_id} | 最新文档={latest_id}" + ) + for version_no, row in enumerate(rows, start=1): + previous_id = rows[version_no - 2].document_id if version_no > 1 else None + latest_mark = "latest" if row.document_id == latest_id else "history" + print( + f" - doc={row.document_id} old_chain={row.version_group_key} old_v={row.version_no} " + f"-> new_v={version_no} prev={previous_id} {latest_mark} type={row.type_name or row.type_id}" + ) + + if len(groups) > limit: + print() + print(f"... 还剩 {len(groups) - limit} 组未显示，可通过 --limit 查看更多") + + +async def apply_groups(conn: asyncpg.Connection, groups: list[list[CandidateRow]]) -> None: + for rows in groups: + root_row = rows[0] + canonical_key = root_row.version_group_key + root_document_id = root_row.document_id + latest_document_id = rows[-1].document_id + + for version_no, row in enumerate(rows, start=1): + previous_id = rows[version_no - 2].document_id if version_no > 1 else None + await conn.execute( + """ + UPDATE leaudit_documents + SET version_group_key = $1, + version_no = $2, + previous_version_id = $3, + root_version_id = $4, + is_latest_version = $5, + updated_at = NOW() + WHERE id = $6 + """, + canonical_key, + version_no, + previous_id, + root_document_id, + row.document_id == latest_document_id, + row.document_id, + ) + + +async def main() -> None: + parser = argparse.ArgumentParser(description="合并历史上被拆开的文档版本链") + parser.add_argument("--region", help="仅处理指定地区") + parser.add_argument("--name", help="仅处理指定 normalized_name") + parser.add_argument("--limit", type=int, default=50, help="最多展示多少组计划") + parser.add_argument("--apply", action="store_true", help="真正写库；默认仅预览") + args = parser.parse_args() + + conn = await asyncpg.connect(load_target_dsn()) + try: + rows = await fetch_candidates( + conn, + region=args.region, + normalized_name=args.name, + ) + groups = group_candidates(rows) + print_plan(groups, max(1, args.limit)) + + if not args.apply: + print() + print("dry-run complete; rerun with --apply to write data") + return + + async with conn.transaction(): + await apply_groups(conn, groups) + + print() + print(f"apply complete; merged groups: {len(groups)}") + finally: + await conn.close() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/scripts/preview_document_version_merge.py b/scripts/preview_document_version_merge.py new file mode 100644 index 0000000..54204ce --- /dev/null +++ b/scripts/preview_document_version_merge.py @@ -0,0 +1,236 @@ +#!/usr/bin/env python3 +"""Preview split document version chains that can be merged under the new rule. + +The new rule groups versions by: +- region +- root business group (一级分组) +- normalized_name + +This script is read-only. It prints candidate groups whose records currently +span more than one version_group_key, which means historical uploads were +split into multiple chains under the old same-type-only rule. +""" + +from __future__ import annotations + +import argparse +import asyncio +from collections import defaultdict +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import asyncpg + + +ROOT = Path(__file__).resolve().parents[1] +APP_TOML = ROOT / "app.toml" + + +@dataclass +class CandidateRow: + document_id: int + region: str + normalized_name: str + root_group_id: int | None + root_group_name: str | None + type_id: int | None + type_name: str | None + group_id: int | None + group_name: str | None + version_group_key: str + version_no: int + is_latest_version: bool + file_name: str | None + updated_at: Any + created_at: Any + + +def load_target_dsn() -> str: + try: + import tomllib + except ImportError: # pragma: no cover + import tomli as tomllib + + with APP_TOML.open("rb") as fh: + config = tomllib.load(fh) + db = config["DB"] + return ( + f"postgresql://{db['USER']}:{db['PASSWORD']}" + f"@{db['HOST']}:{db['PORT']}/{db['NAME']}" + ) + + +async def fetch_candidates( + conn: asyncpg.Connection, + *, + region: str | None, + normalized_name: str | None, +) -> list[CandidateRow]: + filters = ["d.deleted_at IS NULL"] + params: list[Any] = [] + + if region: + params.append(region) + filters.append(f"d.region = ${len(params)}") + if normalized_name: + params.append(normalized_name.strip().lower()) + filters.append(f"d.normalized_name = ${len(params)}") + + where_clause = " AND ".join(filters) + + rows = await conn.fetch( + f""" + WITH doc_scope AS ( + SELECT + d.id AS document_id, + d.region, + d.normalized_name, + d.version_group_key, + d.version_no, + d.is_latest_version, + d.type_id, + dt.name AS type_name, + d.group_id, + child.name AS group_name, + COALESCE( + CASE + WHEN child.id IS NULL THEN NULL + WHEN COALESCE(child.pid, 0) = 0 THEN child.id + ELSE child.pid + END, + inferred.root_group_id + ) AS root_group_id, + COALESCE( + root.name, + inferred.root_group_name + ) AS root_group_name, + d.created_at, + d.updated_at, + f.file_name + FROM leaudit_documents d + LEFT JOIN leaudit_document_types dt + ON dt.id = d.type_id + LEFT JOIN leaudit_evaluation_point_groups child + ON child.id = d.group_id + LEFT JOIN leaudit_evaluation_point_groups root + ON root.id = CASE + WHEN child.id IS NULL THEN NULL + WHEN COALESCE(child.pid, 0) = 0 THEN child.id + ELSE child.pid + END + LEFT JOIN ( + SELECT + child2.document_type_id, + CASE WHEN COUNT(DISTINCT child2.pid) = 1 THEN MIN(child2.pid) END AS root_group_id, + CASE WHEN COUNT(DISTINCT child2.pid) = 1 THEN MIN(parent.name) END AS root_group_name + FROM leaudit_evaluation_point_groups child2 + JOIN leaudit_evaluation_point_groups parent + ON parent.id = child2.pid + WHERE COALESCE(child2.pid, 0) <> 0 + AND child2.deleted_at IS NULL + AND child2.is_enabled = true + AND child2.document_type_id IS NOT NULL + GROUP BY child2.document_type_id + ) inferred + ON inferred.document_type_id = d.type_id + LEFT JOIN leaudit_document_files f + ON f.document_id = d.id + AND f.is_active = true + AND f.file_role = 'primary' + WHERE {where_clause} + ), + conflict_keys AS ( + SELECT + region, + normalized_name, + root_group_id + FROM doc_scope + WHERE normalized_name IS NOT NULL + AND normalized_name <> '' + AND root_group_id IS NOT NULL + GROUP BY region, normalized_name, root_group_id + HAVING COUNT(DISTINCT version_group_key) > 1 + ) + SELECT ds.* + FROM doc_scope ds + JOIN conflict_keys ck + ON ck.region = ds.region + AND ck.normalized_name = ds.normalized_name + AND ck.root_group_id = ds.root_group_id + ORDER BY ds.region, ds.root_group_name, ds.normalized_name, ds.created_at ASC, ds.document_id ASC + """, + *params, + ) + + return [CandidateRow(**dict(row)) for row in rows] + + +def print_preview(rows: list[CandidateRow], limit: int) -> None: + grouped: dict[tuple[str, str, int | None], list[CandidateRow]] = defaultdict(list) + for row in rows: + grouped[(row.region, row.normalized_name, row.root_group_id)].append(row) + + items = sorted( + grouped.items(), + key=lambda item: ( + item[0][0], + item[1][0].root_group_name or "", + item[0][1], + ), + ) + + print(f"候选合并组: {len(items)}") + if not items: + return + + shown = 0 + for (_, _, _), group_rows in items: + if shown >= limit: + break + shown += 1 + + sample = group_rows[0] + distinct_chain_keys = sorted({row.version_group_key for row in group_rows}) + print() + print( + f"[{shown}] 地区={sample.region} | 一级分组={sample.root_group_name or sample.root_group_id} | " + f"归一化名称={sample.normalized_name} | 当前链数={len(distinct_chain_keys)} | 文档数={len(group_rows)}" + ) + + for idx, row in enumerate(group_rows, start=1): + latest_mark = "latest" if row.is_latest_version else "history" + display_name = row.file_name or row.normalized_name + print( + f" - #{idx} doc={row.document_id} chain={row.version_group_key} v{row.version_no} {latest_mark} " + f"type={row.type_name or row.type_id} child={row.group_name or row.group_id} " + f"time={row.updated_at} file={display_name}" + ) + + if len(items) > limit: + print() + print(f"... 还剩 {len(items) - limit} 组未显示，可通过 --limit 查看更多") + + +async def main() -> None: + parser = argparse.ArgumentParser(description="预览需要合并的历史文档版本链") + parser.add_argument("--region", help="仅查看指定地区，如 cz / mz / default") + parser.add_argument("--name", help="仅查看指定 normalized_name") + parser.add_argument("--limit", type=int, default=50, help="最多展示多少组候选，默认 50") + args = parser.parse_args() + + conn = await asyncpg.connect(load_target_dsn()) + try: + rows = await fetch_candidates( + conn, + region=args.region, + normalized_name=args.name, + ) + finally: + await conn.close() + + print_preview(rows, max(1, args.limit)) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/scripts/schema_add_rag_chat.sql b/scripts/schema_add_rag_chat.sql new file mode 100644 index 0000000..3ea3e32 --- /dev/null +++ b/scripts/schema_add_rag_chat.sql @@ -0,0 +1,187 @@ +BEGIN; + +CREATE TABLE IF NOT EXISTS rag_dataset ( + id BIGSERIAL PRIMARY KEY, + name VARCHAR(255) NOT NULL, + description TEXT NOT NULL DEFAULT '', + area VARCHAR(50) NOT NULL DEFAULT '', + is_public BOOLEAN NOT NULL DEFAULT FALSE, + is_default BOOLEAN NOT NULL DEFAULT FALSE, + collection_name VARCHAR(100) NOT NULL UNIQUE, + embedding_model VARCHAR(100) NOT NULL DEFAULT 'text-embedding-v4', + embedding_dim INTEGER NOT NULL DEFAULT 1024, + chunk_max_size INTEGER NOT NULL DEFAULT 800, + chunk_min_size INTEGER NOT NULL DEFAULT 20, + document_count INTEGER NOT NULL DEFAULT 0, + total_chunks INTEGER NOT NULL DEFAULT 0, + retrieval_model JSONB NOT NULL DEFAULT '{}'::jsonb, + sort_order INTEGER NOT NULL DEFAULT 0, + status SMALLINT NOT NULL DEFAULT 1, + created_by BIGINT, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_by BIGINT, + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + deleted_at TIMESTAMPTZ +); + +CREATE TABLE IF NOT EXISTS rag_document ( + id BIGSERIAL PRIMARY KEY, + dataset_id BIGINT NOT NULL REFERENCES rag_dataset(id), + filename VARCHAR(500) NOT NULL, + original_name VARCHAR(500) NOT NULL, + minio_path VARCHAR(1000) NOT NULL, + file_type VARCHAR(20) NOT NULL, + file_size BIGINT NOT NULL DEFAULT 0, + chunk_count INTEGER NOT NULL DEFAULT 0, + indexing_status VARCHAR(20) NOT NULL DEFAULT 'pending', + indexing_error TEXT, + indexing_started_at TIMESTAMPTZ, + indexing_completed_at TIMESTAMPTZ, + enabled BOOLEAN NOT NULL DEFAULT TRUE, + hit_count INTEGER NOT NULL DEFAULT 0, + created_by BIGINT, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + deleted_at TIMESTAMPTZ +); + +CREATE TABLE IF NOT EXISTS rag_chat_app ( + id BIGSERIAL PRIMARY KEY, + name VARCHAR(255) NOT NULL, + description TEXT NOT NULL DEFAULT '', + area VARCHAR(50) NOT NULL DEFAULT '', + dataset_id BIGINT REFERENCES rag_dataset(id), + system_prompt TEXT NOT NULL DEFAULT '', + llm_model VARCHAR(100) NOT NULL DEFAULT '', + temperature REAL NOT NULL DEFAULT 0.3, + max_tokens INTEGER NOT NULL DEFAULT 2048, + opening_statement TEXT NOT NULL DEFAULT '', + suggested_questions TEXT NOT NULL DEFAULT '[]', + is_default BOOLEAN NOT NULL DEFAULT FALSE, + sort_order INTEGER NOT NULL DEFAULT 0, + status SMALLINT NOT NULL DEFAULT 1, + created_by BIGINT, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_by BIGINT, + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + deleted_at TIMESTAMPTZ +); + +CREATE TABLE IF NOT EXISTS rag_conversation ( + id BIGSERIAL PRIMARY KEY, + conversation_id VARCHAR(100) NOT NULL UNIQUE, + user_id BIGINT NOT NULL, + app_id BIGINT REFERENCES rag_chat_app(id), + name VARCHAR(500) NOT NULL DEFAULT '新对话', + introduction TEXT NOT NULL DEFAULT '', + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + deleted_at TIMESTAMPTZ +); + +CREATE TABLE IF NOT EXISTS rag_message ( + id BIGSERIAL PRIMARY KEY, + message_id VARCHAR(100) NOT NULL UNIQUE, + conversation_id VARCHAR(100) NOT NULL REFERENCES rag_conversation(conversation_id), + role VARCHAR(20) NOT NULL, + content TEXT NOT NULL DEFAULT '', + sources JSONB NOT NULL DEFAULT '[]'::jsonb, + metadata JSONB NOT NULL DEFAULT '{}'::jsonb, + feedback VARCHAR(20), + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +ALTER TABLE rag_dataset + ADD COLUMN IF NOT EXISTS retrieval_model JSONB NOT NULL DEFAULT '{}'::jsonb, + ADD COLUMN IF NOT EXISTS embedding_model VARCHAR(100) NOT NULL DEFAULT 'text-embedding-v4', + ADD COLUMN IF NOT EXISTS embedding_dim INTEGER NOT NULL DEFAULT 1024, + ADD COLUMN IF NOT EXISTS chunk_max_size INTEGER NOT NULL DEFAULT 800, + ADD COLUMN IF NOT EXISTS chunk_min_size INTEGER NOT NULL DEFAULT 20, + ADD COLUMN IF NOT EXISTS document_count INTEGER NOT NULL DEFAULT 0, + ADD COLUMN IF NOT EXISTS total_chunks INTEGER NOT NULL DEFAULT 0, + ADD COLUMN IF NOT EXISTS sort_order INTEGER NOT NULL DEFAULT 0, + ADD COLUMN IF NOT EXISTS status SMALLINT NOT NULL DEFAULT 1, + ADD COLUMN IF NOT EXISTS updated_by BIGINT, + ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + ADD COLUMN IF NOT EXISTS deleted_at TIMESTAMPTZ; + +ALTER TABLE rag_document + ADD COLUMN IF NOT EXISTS enabled BOOLEAN NOT NULL DEFAULT TRUE, + ADD COLUMN IF NOT EXISTS hit_count INTEGER NOT NULL DEFAULT 0, + ADD COLUMN IF NOT EXISTS indexing_started_at TIMESTAMPTZ, + ADD COLUMN IF NOT EXISTS indexing_completed_at TIMESTAMPTZ, + ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + ADD COLUMN IF NOT EXISTS deleted_at TIMESTAMPTZ; + +ALTER TABLE rag_chat_app + ADD COLUMN IF NOT EXISTS dataset_id BIGINT REFERENCES rag_dataset(id), + ADD COLUMN IF NOT EXISTS system_prompt TEXT NOT NULL DEFAULT '', + ADD COLUMN IF NOT EXISTS llm_model VARCHAR(100) NOT NULL DEFAULT '', + ADD COLUMN IF NOT EXISTS temperature REAL NOT NULL DEFAULT 0.3, + ADD COLUMN IF NOT EXISTS max_tokens INTEGER NOT NULL DEFAULT 2048, + ADD COLUMN IF NOT EXISTS opening_statement TEXT NOT NULL DEFAULT '', + ADD COLUMN IF NOT EXISTS suggested_questions TEXT NOT NULL DEFAULT '[]', + ADD COLUMN IF NOT EXISTS is_default BOOLEAN NOT NULL DEFAULT FALSE, + ADD COLUMN IF NOT EXISTS sort_order INTEGER NOT NULL DEFAULT 0, + ADD COLUMN IF NOT EXISTS status SMALLINT NOT NULL DEFAULT 1, + ADD COLUMN IF NOT EXISTS updated_by BIGINT, + ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + ADD COLUMN IF NOT EXISTS deleted_at TIMESTAMPTZ; + +ALTER TABLE rag_conversation + ADD COLUMN IF NOT EXISTS app_id BIGINT REFERENCES rag_chat_app(id), + ADD COLUMN IF NOT EXISTS introduction TEXT NOT NULL DEFAULT '', + ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + ADD COLUMN IF NOT EXISTS deleted_at TIMESTAMPTZ; + +ALTER TABLE rag_message + ADD COLUMN IF NOT EXISTS sources JSONB NOT NULL DEFAULT '[]'::jsonb, + ADD COLUMN IF NOT EXISTS metadata JSONB NOT NULL DEFAULT '{}'::jsonb, + ADD COLUMN IF NOT EXISTS feedback VARCHAR(20), + ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(); + +UPDATE rag_dataset SET retrieval_model = '{}'::jsonb WHERE retrieval_model IS NULL; +UPDATE rag_document SET enabled = TRUE WHERE enabled IS NULL; +UPDATE rag_document SET hit_count = 0 WHERE hit_count IS NULL; +UPDATE rag_chat_app SET suggested_questions = '[]' WHERE suggested_questions IS NULL; +UPDATE rag_message SET sources = '[]'::jsonb WHERE sources IS NULL; +UPDATE rag_message SET metadata = '{}'::jsonb WHERE metadata IS NULL; + +CREATE UNIQUE INDEX IF NOT EXISTS uq_rag_dataset_collection_name ON rag_dataset(collection_name); +CREATE INDEX IF NOT EXISTS idx_rag_dataset_area ON rag_dataset(area) WHERE deleted_at IS NULL; +CREATE INDEX IF NOT EXISTS idx_rag_dataset_status ON rag_dataset(status) WHERE deleted_at IS NULL; +CREATE INDEX IF NOT EXISTS idx_rag_document_dataset ON rag_document(dataset_id) WHERE deleted_at IS NULL; +CREATE INDEX IF NOT EXISTS idx_rag_document_status ON rag_document(indexing_status) WHERE deleted_at IS NULL; +CREATE INDEX IF NOT EXISTS idx_rag_document_original_name ON rag_document(dataset_id, original_name) WHERE deleted_at IS NULL; +CREATE INDEX IF NOT EXISTS idx_rag_chat_app_area ON rag_chat_app(area) WHERE deleted_at IS NULL; +CREATE INDEX IF NOT EXISTS idx_rag_chat_app_status ON rag_chat_app(status) WHERE deleted_at IS NULL; +CREATE INDEX IF NOT EXISTS idx_rag_conversation_user ON rag_conversation(user_id) WHERE deleted_at IS NULL; +CREATE INDEX IF NOT EXISTS idx_rag_conversation_app ON rag_conversation(app_id) WHERE deleted_at IS NULL; +CREATE INDEX IF NOT EXISTS idx_rag_message_conversation ON rag_message(conversation_id, created_at); + +CREATE OR REPLACE FUNCTION update_rag_updated_at() +RETURNS TRIGGER AS $$ +BEGIN + NEW.updated_at = NOW(); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +DO $$ +DECLARE + t TEXT; +BEGIN + FOREACH t IN ARRAY ARRAY['rag_dataset', 'rag_document', 'rag_chat_app', 'rag_conversation'] + LOOP + EXECUTE format('DROP TRIGGER IF EXISTS trg_%s_updated_at ON %I', t, t); + EXECUTE format( + 'CREATE TRIGGER trg_%s_updated_at + BEFORE UPDATE ON %I + FOR EACH ROW EXECUTE FUNCTION update_rag_updated_at()', + t, t + ); + END LOOP; +END; +$$; + +COMMIT; diff --git a/scripts/user_rbac_seed.sql b/scripts/user_rbac_seed.sql index 0cf8faa..8c14aa0 100644 --- a/scripts/user_rbac_seed.sql +++ b/scripts/user_rbac_seed.sql @@ -48,8 +48,8 @@ VALUES ON CONFLICT DO NOTHING; UPDATE role_route -SET deleted_at = NOW(), updated_at = NOW() -WHERE deleted_at IS NULL +SET status = 0, updated_at = NOW() +WHERE status <> 0 AND route_id IN ( SELECT id FROM sys_routes WHERE route_path = '/rules/sets' AND deleted_at IS NULL ); @@ -96,15 +96,15 @@ VALUES ('evaluation_point:create:write', 'evaluation_point', 'create', 'write', '创建评查点', '创建评查点', 'API', TRUE, NULL, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, NULL, NULL, NULL, 54, NULL, '/api/v3/evaluation-points', 'POST', NULL), ('evaluation_point:update:write', 'evaluation_point', 'update', 'write', '更新评查点', '更新评查点', 'API', TRUE, NULL, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, NULL, NULL, NULL, 55, NULL, '/api/v3/evaluation-points/{id}', 'PUT', NULL), ('evaluation_point:delete:delete', 'evaluation_point', 'delete', 'delete', '删除评查点', '删除评查点', 'API', TRUE, NULL, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, NULL, NULL, NULL, 56, NULL, '/api/v3/evaluation-points/{id}', 'DELETE', NULL), - ('cross_review:task:create', 'cross_review', 'task', 'create', '创建交叉评查任务', '创建交叉评查任务', 'API', TRUE, NULL, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, NULL, NULL, NULL, 57, NULL, '/api/v3/cross-review/tasks', 'POST', ARRAY['/cross-checking/upload']), - ('cross_review:task:read', 'cross_review', 'task', 'read', '查看交叉评查任务', '查看交叉评查任务', 'API', TRUE, NULL, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, NULL, NULL, NULL, 58, NULL, '/api/v3/cross-review/tasks/query', 'POST', ARRAY['/cross-checking']), - ('cross_review:progress:view', 'cross_review', 'progress', 'view', '查看交叉评查任务进度', '查看交叉评查任务进度', 'API', TRUE, NULL, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, NULL, NULL, NULL, 59, NULL, '/api/v3/cross-review/tasks/{task_id}/progress', 'GET', ARRAY['/cross-checking']), - ('cross_review:document:read', 'cross_review', 'document', 'read', '查看交叉评查任务文档', '查看交叉评查任务文档', 'API', TRUE, NULL, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, NULL, NULL, NULL, 60, NULL, '/api/v3/cross-review/tasks/{task_id}/documents', 'GET', ARRAY['/cross-checking','/cross-checking/result']), - ('cross_review:document:complete', 'cross_review', 'document', 'complete', '确认交叉评查文档完成', '确认交叉评查文档完成', 'API', TRUE, NULL, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, NULL, NULL, NULL, 61, NULL, '/api/v3/cross-review/tasks/{task_id}/can-confirm', 'GET', ARRAY['/cross-checking/result']), - ('cross_review:proposal:create', 'cross_review', 'proposal', 'create', '创建交叉评查提案', '创建交叉评查提案', 'API', TRUE, NULL, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, NULL, NULL, NULL, 62, NULL, '/api/v3/cross-review/proposals', 'POST', ARRAY['/cross-checking/result']), - ('cross_review:proposal:read', 'cross_review', 'proposal', 'read', '查看交叉评查提案', '查看交叉评查提案', 'API', TRUE, NULL, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, NULL, NULL, NULL, 63, NULL, '/api/v3/cross-review/documents/{document_id}/proposals', 'GET', ARRAY['/cross-checking/result']), - ('cross_review:proposal:delete', 'cross_review', 'proposal', 'delete', '撤销交叉评查提案', '撤销交叉评查提案', 'API', TRUE, NULL, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, NULL, NULL, NULL, 64, NULL, '/api/v3/cross-review/proposals/{proposal_id}', 'DELETE', ARRAY['/cross-checking/result']), - ('cross_review:proposal:vote', 'cross_review', 'proposal', 'vote', '交叉评查提案投票', '交叉评查提案投票', 'API', TRUE, NULL, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, NULL, NULL, NULL, 65, NULL, '/api/v3/cross-review/proposals/{proposal_id}/votes', 'POST', ARRAY['/cross-checking/result']), + ('cross_review:task:create', 'cross_review', 'task', 'create', '创建交叉评查任务', '创建交叉评查任务', 'API', TRUE, NULL, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, NULL, NULL, NULL, 57, NULL, '/api/v3/cross-review/tasks', 'POST', NULL), + ('cross_review:task:read', 'cross_review', 'task', 'read', '查看交叉评查任务', '查看交叉评查任务', 'API', TRUE, NULL, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, NULL, NULL, NULL, 58, NULL, '/api/v3/cross-review/tasks/query', 'POST', NULL), + ('cross_review:progress:view', 'cross_review', 'progress', 'view', '查看交叉评查任务进度', '查看交叉评查任务进度', 'API', TRUE, NULL, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, NULL, NULL, NULL, 59, NULL, '/api/v3/cross-review/tasks/{task_id}/progress', 'GET', NULL), + ('cross_review:document:read', 'cross_review', 'document', 'read', '查看交叉评查任务文档', '查看交叉评查任务文档', 'API', TRUE, NULL, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, NULL, NULL, NULL, 60, NULL, '/api/v3/cross-review/tasks/{task_id}/documents', 'GET', NULL), + ('cross_review:document:complete', 'cross_review', 'document', 'complete', '确认交叉评查文档完成', '确认交叉评查文档完成', 'API', TRUE, NULL, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, NULL, NULL, NULL, 61, NULL, '/api/v3/cross-review/tasks/{task_id}/can-confirm', 'GET', NULL), + ('cross_review:proposal:create', 'cross_review', 'proposal', 'create', '创建交叉评查提案', '创建交叉评查提案', 'API', TRUE, NULL, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, NULL, NULL, NULL, 62, NULL, '/api/v3/cross-review/proposals', 'POST', NULL), + ('cross_review:proposal:read', 'cross_review', 'proposal', 'read', '查看交叉评查提案', '查看交叉评查提案', 'API', TRUE, NULL, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, NULL, NULL, NULL, 63, NULL, '/api/v3/cross-review/documents/{document_id}/proposals', 'GET', NULL), + ('cross_review:proposal:delete', 'cross_review', 'proposal', 'delete', '撤销交叉评查提案', '撤销交叉评查提案', 'API', TRUE, NULL, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, NULL, NULL, NULL, 64, NULL, '/api/v3/cross-review/proposals/{proposal_id}', 'DELETE', NULL), + ('cross_review:proposal:vote', 'cross_review', 'proposal', 'vote', '交叉评查提案投票', '交叉评查提案投票', 'API', TRUE, NULL, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, NULL, NULL, NULL, 65, NULL, '/api/v3/cross-review/proposals/{proposal_id}/votes', 'POST', NULL), ('users:list:read', 'users', 'list', 'read', '查看用户列表', '用户列表', 'API', TRUE, NULL, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, NULL, NULL, NULL, 70, NULL, '/api/users/list', 'GET', NULL), ('users:create:write', 'users', 'create', 'write', '创建用户', '创建用户', 'API', TRUE, NULL, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, NULL, NULL, NULL, 71, NULL, '/api/users', 'POST', NULL), @@ -116,7 +116,14 @@ VALUES ('rbac:roles:update', 'rbac', 'roles', 'update', '维护角色信息', '维护角色', 'API', TRUE, NULL, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, NULL, NULL, NULL, 81, NULL, '/api/rbac/roles/{role_id}', 'PUT', NULL), ('rbac:permissions:read', 'rbac', 'permissions', 'read', '查看权限点列表', '权限点列表', 'API', TRUE, NULL, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, NULL, NULL, NULL, 82, NULL, '/api/rbac/permissions', 'GET', NULL), ('rbac:role_permissions:write', 'rbac', 'role_permissions', 'write', '分配角色权限', '分配角色权限', 'API', TRUE, NULL, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, NULL, NULL, NULL, 83, NULL, '/api/rbac/roles/{role_id}/permissions', 'POST', NULL), - ('rbac:role_routes:write', 'rbac', 'role_routes', 'write', '分配角色菜单', '分配角色菜单', 'API', TRUE, NULL, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, NULL, NULL, NULL, 84, NULL, '/api/rbac/roles/{role_id}/routes', 'PUT', NULL) + ('rbac:role_routes:write', 'rbac', 'role_routes', 'write', '分配角色菜单', '分配角色菜单', 'API', TRUE, NULL, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, NULL, NULL, NULL, 84, NULL, '/api/rbac/roles/{role_id}/routes', 'PUT', NULL), + ('rag:app:read', 'rag', 'app', 'read', '查看 RAG 应用', '查看 RAG 应用', 'API', TRUE, NULL, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, NULL, NULL, NULL, 85, NULL, '/api/v3/rag/apps', 'GET', NULL), + ('rag:chat:use', 'rag', 'chat', 'use', '使用 RAG 对话', '使用 RAG 对话', 'API', TRUE, NULL, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, NULL, NULL, NULL, 86, NULL, '/api/v3/rag/chat/messages', 'POST', NULL), + ('rag:conversation:read', 'rag', 'conversation', 'read', '查看 RAG 会话', '查看 RAG 会话', 'API', TRUE, NULL, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, NULL, NULL, NULL, 87, NULL, '/api/v3/rag/chat/conversations', 'GET', NULL), + ('rag:conversation:update', 'rag', 'conversation', 'update', '重命名 RAG 会话', '重命名 RAG 会话', 'API', TRUE, NULL, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, NULL, NULL, NULL, 88, NULL, '/api/v3/rag/chat/conversations/{ConversationId}', 'PATCH', NULL), + ('rag:conversation:delete', 'rag', 'conversation', 'delete', '删除 RAG 会话', '删除 RAG 会话', 'API', TRUE, NULL, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, NULL, NULL, NULL, 89, NULL, '/api/v3/rag/chat/conversations/{ConversationId}', 'DELETE', NULL), + ('rag:message:feedback', 'rag', 'message', 'feedback', '反馈 RAG 消息', '反馈 RAG 消息', 'API', TRUE, NULL, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, NULL, NULL, NULL, 90, NULL, '/api/v3/rag/chat/messages/{MessageId}/feedback', 'POST', NULL), + ('rag:dataset:read', 'rag', 'dataset', 'read', '查看 RAG 知识库', '查看 RAG 知识库', 'API', TRUE, NULL, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, NULL, NULL, NULL, 91, NULL, '/api/v3/rag/datasets/my', 'GET', NULL) ON CONFLICT (permission_key) DO UPDATE SET module = EXCLUDED.module, resource = EXCLUDED.resource, @@ -258,6 +265,13 @@ seed(role_key, permission_key, grant_type, data_scope) AS ( ('super_admin', 'users:roles_assign:write', 'GRANT', 'ALL'), ('super_admin', 'rbac:roles:read', 'GRANT', 'ALL'), ('super_admin', 'rbac:roles:update', 'GRANT', 'ALL'), + ('super_admin', 'rag:app:read', 'GRANT', 'ALL'), + ('super_admin', 'rag:chat:use', 'GRANT', 'ALL'), + ('super_admin', 'rag:conversation:read', 'GRANT', 'ALL'), + ('super_admin', 'rag:conversation:update', 'GRANT', 'ALL'), + ('super_admin', 'rag:conversation:delete', 'GRANT', 'ALL'), + ('super_admin', 'rag:message:feedback', 'GRANT', 'ALL'), + ('super_admin', 'rag:dataset:read', 'GRANT', 'ALL'), ('super_admin', 'rbac:permissions:read', 'GRANT', 'ALL'), ('super_admin', 'rbac:role_permissions:write', 'GRANT', 'ALL'), ('super_admin', 'rbac:role_routes:write', 'GRANT', 'ALL'), @@ -304,6 +318,13 @@ seed(role_key, permission_key, grant_type, data_scope) AS ( ('provincial_admin', 'users:roles_assign:write', 'GRANT', 'ALL'), ('provincial_admin', 'rbac:roles:read', 'GRANT', 'ALL'), ('provincial_admin', 'rbac:roles:update', 'GRANT', 'ALL'), + ('provincial_admin', 'rag:app:read', 'GRANT', 'ALL'), + ('provincial_admin', 'rag:chat:use', 'GRANT', 'ALL'), + ('provincial_admin', 'rag:conversation:read', 'GRANT', 'ALL'), + ('provincial_admin', 'rag:conversation:update', 'GRANT', 'ALL'), + ('provincial_admin', 'rag:conversation:delete', 'GRANT', 'ALL'), + ('provincial_admin', 'rag:message:feedback', 'GRANT', 'ALL'), + ('provincial_admin', 'rag:dataset:read', 'GRANT', 'ALL'), ('provincial_admin', 'rbac:permissions:read', 'GRANT', 'ALL'), ('provincial_admin', 'rbac:role_permissions:write', 'GRANT', 'ALL'), ('provincial_admin', 'rbac:role_routes:write', 'GRANT', 'ALL'), @@ -339,6 +360,13 @@ seed(role_key, permission_key, grant_type, data_scope) AS ( ('admin', 'evaluation_point:create:write', 'GRANT', 'DEPT'), ('admin', 'evaluation_point:update:write', 'GRANT', 'DEPT'), ('admin', 'evaluation_point:delete:delete', 'GRANT', 'DEPT'), + ('admin', 'rag:app:read', 'GRANT', 'DEPT'), + ('admin', 'rag:chat:use', 'GRANT', 'DEPT'), + ('admin', 'rag:conversation:read', 'GRANT', 'DEPT'), + ('admin', 'rag:conversation:update', 'GRANT', 'DEPT'), + ('admin', 'rag:conversation:delete', 'GRANT', 'DEPT'), + ('admin', 'rag:message:feedback', 'GRANT', 'DEPT'), + ('admin', 'rag:dataset:read', 'GRANT', 'DEPT'), ('admin', 'users:list:read', 'GRANT', 'DEPT'), ('admin', 'users:update:write', 'GRANT', 'DEPT'), @@ -356,7 +384,14 @@ seed(role_key, permission_key, grant_type, data_scope) AS ( ('common', 'rules:list:read', 'GRANT', 'DEPT'), ('common', 'rules:version_list:read', 'GRANT', 'DEPT'), ('common', 'rules:content:read', 'GRANT', 'DEPT'), - ('common', 'rules:binding_list:read', 'GRANT', 'DEPT') + ('common', 'rules:binding_list:read', 'GRANT', 'DEPT'), + ('common', 'rag:app:read', 'GRANT', 'SELF'), + ('common', 'rag:chat:use', 'GRANT', 'SELF'), + ('common', 'rag:conversation:read', 'GRANT', 'SELF'), + ('common', 'rag:conversation:update', 'GRANT', 'SELF'), + ('common', 'rag:conversation:delete', 'GRANT', 'SELF'), + ('common', 'rag:message:feedback', 'GRANT', 'SELF'), + ('common', 'rag:dataset:read', 'GRANT', 'SELF') ) INSERT INTO role_permissions (role_id, permission_id, grant_type, data_scope, created_at, updated_at) SELECT rm.id, pm.id, s.grant_type, s.data_scope, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP