TanWenyan/leaudit-platform-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add contract template v3 api and legacy oss migration

Browse Source
This commit is contained in:
wren
2026-05-19 18:38:17 +08:00
parent afaba4dd99
commit 16e8668150
11 changed files with 1241 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/migrate_legacy_contract_templates.py
+374
View File
@@ -0,0 +1,374 @@
#!/usr/bin/env python3
"""Migrate legacy contract templates from docauditai to leaudit_platform."""
from __future__ import annotations
import argparse
import asyncio
from dataclasses import dataclass
from io import BytesIO
from pathlib import Path
import asyncpg
from minio import Minio
from fastapi_common.fastapi_common_storage.oss_path_utils import OssPathUtils
ROOT = Path(__file__).resolve().parents[1]
APP_TOML = ROOT / "app.toml"
OLD_BUCKET = "docauditai"
@dataclass(frozen=True)
class LegacyCategory:
id: int
name: str
icon: str | None
description: str | None
sort_order: int
@dataclass(frozen=True)
class LegacyTemplate:
id: int
template_code: str
title: str
category_id: int
description: str | None
file_path: str | None
file_format: str | None
is_featured: bool | None
created_at: object
updated_at: object
pdf_file_path: str | None
category_name: str
def load_target_config() -> dict[str, str]:
try:
import tomllib
except ImportError: # pragma: no cover
import tomli as tomllib
with APP_TOML.open("rb") as fh:
config = tomllib.load(fh)
db = config["DB"]
oss = config["OSS"]
return {
"target_dsn": (
f"postgresql://{db['USER']}:{db['PASSWORD']}"
f"@{db['HOST']}:{db['PORT']}/{db['NAME']}"
),
"oss_endpoint": oss["ENDPOINT"],
"oss_base_url": oss.get("BASE_URL", ""),
"oss_access_key": oss["ACCESS_KEY"],
"oss_secret_key": oss["SECRET_KEY"],
"oss_bucket": oss["BUCKET"],
}
def build_legacy_dsn(args: argparse.Namespace) -> str:
return (
f"postgresql://{args.legacy_user}:{args.legacy_password}"
f"@{args.legacy_host}:{args.legacy_port}/{args.legacy_db}"
)
def build_minio_client(config: dict[str, str]) -> Minio:
endpoint = config["oss_endpoint"]
base_url = config.get("oss_base_url", "")
if base_url.startswith("http://"):
secure = False
elif base_url.startswith("https://"):
secure = True
else:
secure = endpoint.startswith("https://")
host = endpoint.replace("http://", "").replace("https://", "")
return Minio(
host,
access_key=config["oss_access_key"],
secret_key=config["oss_secret_key"],
secure=secure,
)
async def fetch_legacy_categories(conn: asyncpg.Connection) -> list[LegacyCategory]:
rows = await conn.fetch(
"""
SELECT id, name, icon, description, COALESCE(sort_order, 0) AS sort_order
FROM public.contract_categories
ORDER BY id
"""
)
return [LegacyCategory(**dict(row)) for row in rows]
async def fetch_legacy_templates(conn: asyncpg.Connection) -> list[LegacyTemplate]:
rows = await conn.fetch(
"""
SELECT
t.id,
t.template_code,
t.title,
t.category_id,
t.description,
t.file_path,
t.file_format,
t.is_featured,
t.created_at,
t.updated_at,
t.pdf_file_path,
c.name AS category_name
FROM public.contract_templates t
LEFT JOIN public.contract_categories c ON c.id = t.category_id
ORDER BY t.id
"""
)
return [LegacyTemplate(**dict(row)) for row in rows]
def resolve_docx_path(template: LegacyTemplate, object_keys: set[str]) -> str:
file_path = (template.file_path or "").strip()
if not file_path:
raise ValueError(f"template {template.id} missing file_path")
if file_path in object_keys:
pdf_path = (template.pdf_file_path or "").strip()
if pdf_path and pdf_path in object_keys:
expected_docx = str(Path(pdf_path).with_suffix(".docx"))
if expected_docx in object_keys:
current_name = Path(file_path).name
expected_name = Path(expected_docx).name
if current_name != expected_name:
return expected_docx
return file_path
pdf_path = (template.pdf_file_path or "").strip()
if pdf_path:
expected_docx = str(Path(pdf_path).with_suffix(".docx"))
if expected_docx in object_keys:
return expected_docx
raise FileNotFoundError(f"template {template.id} docx not found: {file_path}")
def resolve_pdf_path(template: LegacyTemplate, object_keys: set[str]) -> str:
pdf_path = (template.pdf_file_path or "").strip()
if not pdf_path:
raise ValueError(f"template {template.id} missing pdf_file_path")
if pdf_path in object_keys:
return pdf_path
raise FileNotFoundError(f"template {template.id} pdf not found: {pdf_path}")
def build_new_object_keys(template: LegacyTemplate, docx_path: str, pdf_path: str) -> tuple[str, str]:
docx_key = OssPathUtils.BuildContractTemplateKey(
CategoryName=template.category_name,
TemplateCode=template.template_code,
FileRole="source",
FileName=Path(docx_path).name,
)
pdf_key = OssPathUtils.BuildContractTemplateKey(
CategoryName=template.category_name,
TemplateCode=template.template_code,
FileRole="preview",
FileName=Path(pdf_path).name,
)
return docx_key, pdf_key
def copy_object_bytes(
client: Minio,
*,
source_bucket: str,
source_key: str,
target_bucket: str,
target_key: str,
) -> None:
response = client.get_object(source_bucket, source_key)
try:
payload = response.read()
finally:
response.close()
response.release_conn()
client.put_object(
target_bucket,
target_key,
data=BytesIO(payload),
length=len(payload),
)
def ensure_bucket(client: Minio, bucket: str) -> None:
if not client.bucket_exists(bucket):
client.make_bucket(bucket)
async def reset_target_tables(conn: asyncpg.Connection) -> None:
await conn.execute("TRUNCATE TABLE public.contract_templates RESTART IDENTITY CASCADE")
await conn.execute("TRUNCATE TABLE public.contract_categories RESTART IDENTITY CASCADE")
async def insert_categories(conn: asyncpg.Connection, categories: list[LegacyCategory]) -> None:
for category in categories:
await conn.execute(
"""
INSERT INTO public.contract_categories (id, name, icon, description, sort_order)
VALUES ($1, $2, $3, $4, $5)
""",
category.id,
category.name,
category.icon,
category.description,
category.sort_order,
)
await conn.execute(
"""
SELECT setval(
pg_get_serial_sequence('public.contract_categories', 'id'),
COALESCE((SELECT MAX(id) FROM public.contract_categories), 1),
TRUE
)
"""
)
async def insert_templates(
conn: asyncpg.Connection,
templates: list[LegacyTemplate],
template_paths: dict[int, tuple[str, str]],
) -> None:
for template in templates:
file_path, pdf_file_path = template_paths[template.id]
await conn.execute(
"""
INSERT INTO public.contract_templates (
id,
template_code,
title,
category_id,
description,
file_path,
file_format,
is_featured,
created_at,
updated_at,
pdf_file_path
)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
""",
template.id,
template.template_code,
template.title,
template.category_id,
template.description,
file_path,
(template.file_format or "docx").lower(),
bool(template.is_featured),
template.created_at,
template.updated_at,
pdf_file_path,
)
await conn.execute(
"""
SELECT setval(
pg_get_serial_sequence('public.contract_templates', 'id'),
COALESCE((SELECT MAX(id) FROM public.contract_templates), 1),
TRUE
)
"""
)
async def main() -> None:
parser = argparse.ArgumentParser(description="Migrate legacy contract templates.")
parser.add_argument("--legacy-host", default="nas.7bm.co")
parser.add_argument("--legacy-port", type=int, default=54302)
parser.add_argument("--legacy-db", default="docauditai")
parser.add_argument("--legacy-user", default="root")
parser.add_argument("--legacy-password", default="postgresql.2025.qwe")
parser.add_argument("--apply", action="store_true", help="Apply migration to OSS and target DB.")
args = parser.parse_args()
config = load_target_config()
legacy_dsn = build_legacy_dsn(args)
target_dsn = config["target_dsn"]
target_bucket = config["oss_bucket"]
minio_client = build_minio_client(config)
legacy_conn = await asyncpg.connect(legacy_dsn)
target_conn = await asyncpg.connect(target_dsn)
try:
ensure_bucket(minio_client, target_bucket)
categories = await fetch_legacy_categories(legacy_conn)
templates = await fetch_legacy_templates(legacy_conn)
object_keys = {
obj.object_name
for obj in minio_client.list_objects(OLD_BUCKET, prefix="contract-template/", recursive=True)
}
template_paths: dict[int, tuple[str, str]] = {}
for template in templates:
docx_path = resolve_docx_path(template, object_keys)
pdf_path = resolve_pdf_path(template, object_keys)
template_paths[template.id] = build_new_object_keys(template, docx_path, pdf_path)
print(f"legacy categories: {len(categories)}")
print(f"legacy templates: {len(templates)}")
for template in templates:
old_docx = resolve_docx_path(template, object_keys)
old_pdf = resolve_pdf_path(template, object_keys)
new_docx, new_pdf = template_paths[template.id]
print(
f"[{template.id}] {template.template_code} | "
f"{old_docx} -> {new_docx} | {old_pdf} -> {new_pdf}"
)
if not args.apply:
print("dry-run complete; rerun with --apply to execute migration")
return
if args.apply:
found_correction = False
for template in templates:
old_docx = resolve_docx_path(template, object_keys)
old_pdf = resolve_pdf_path(template, object_keys)
new_docx, new_pdf = template_paths[template.id]
if old_docx != (template.file_path or "").strip():
print(
f"corrected docx path for template {template.id}: "
f"{template.file_path} -> {old_docx}"
)
found_correction = True
copy_object_bytes(
minio_client,
source_bucket=OLD_BUCKET,
source_key=old_docx,
target_bucket=target_bucket,
target_key=new_docx,
)
copy_object_bytes(
minio_client,
source_bucket=OLD_BUCKET,
source_key=old_pdf,
target_bucket=target_bucket,
target_key=new_pdf,
)
if not found_correction:
print("no legacy path corrections required")
async with target_conn.transaction():
await reset_target_tables(target_conn)
await insert_categories(target_conn, categories)
await insert_templates(target_conn, templates, template_paths)
print("migration applied successfully")
finally:
await legacy_conn.close()
await target_conn.close()
if __name__ == "__main__":
asyncio.run(main())
scripts/创建sql/README.md
+90
View File
@@ -46,6 +46,11 @@ psql -h <host> -U <user> -d <db_name> -v ON_ERROR_STOP=1 -f scripts/创建sql/<f
3. `schema_v2_add_evaluation_tables.sql`
4. `seed_home_entry_modules.sql`
### 合同模板模块上线
1. `schema_contract_templates.sql`
2. `seed_contract_templates_rbac.sql`
### 系统使用统计上线
1. `schema_add_usage_stats.sql`
@@ -221,6 +226,91 @@ ORDER BY permission_key;
1. `schema_v3_add_cross_review_phase1.sql`
2. `seed_cross_review_phase1_permissions.sql`
### 七、合同模板
- `schema_contract_templates.sql`
- 用途：在主库创建合同模板分类表和模板表
- 主要内容：新增 `contract_categories`、`contract_templates`、补索引和注释
- 执行时机：上线合同模板新后端接口前必跑
- `seed_contract_templates_rbac.sql`
- 用途：补齐合同模板只读权限点
- 主要内容：新增 `contract_template:list:read`、`contract_template:search:read`、`contract_template:detail:read`
- 依赖：`sys_routes` 中已经存在 `/contract-template/list` 和 `/contract-template/search`
- `migrate_legacy_contract_templates.py`
- 用途：把老库 `docauditai` 的合同模板分类、模板记录和旧 bucket 文件迁入主库 `leaudit_platform`
- 主要内容：
- 读取老库 `public.contract_categories`、`public.contract_templates`
- 从旧 bucket `docauditai` 读取 `contract-template/...` 对象
- 复制到新 bucket `leaudit` 的 `contract-templates/...` 相对路径
- 回写主库 `contract_categories`、`contract_templates.file_path`、`contract_templates.pdf_file_path`
- 适用场景：主库已完成建表与权限初始化，但仍是 demo 数据或空数据时
- 注意：
- 脚本会重置主库 `contract_categories` / `contract_templates` 当前数据并按老库正式数据重建
- 当前已知会自动修正 1 条老脏数据：
- `contract_templates.id=3`
- 标题：`房屋租赁合同（我方承租）`
- 老 `file_path` 误指向“我方出租”docx，迁移时会自动改成“我方承租”docx
#### 推荐顺序
1. `schema_contract_templates.sql`
2. `seed_contract_templates_rbac.sql`
3. `python scripts/migrate_legacy_contract_templates.py`
4. `python scripts/migrate_legacy_contract_templates.py --apply`
#### 标准执行命令
```bash
psql -h <host> -U <user> -d <db_name> -v ON_ERROR_STOP=1 -f scripts/创建sql/schema_contract_templates.sql
psql -h <host> -U <user> -d <db_name> -v ON_ERROR_STOP=1 -f scripts/创建sql/seed_contract_templates_rbac.sql
# 先 dry-run，看旧路径 -> 新路径映射
python scripts/migrate_legacy_contract_templates.py
# 确认无误后正式执行：复制 OSS 文件 + 回写主库
python scripts/migrate_legacy_contract_templates.py --apply
```
#### 执行后验收
```sql
SELECT to_regclass('public.contract_categories');
SELECT to_regclass('public.contract_templates');
SELECT permission_key
FROM permissions
WHERE permission_key LIKE 'contract_template:%'
ORDER BY permission_key;
SELECT r.role_key, p.permission_key, rp.grant_type, rp.data_scope
FROM role_permissions rp
JOIN roles r ON r.id = rp.role_id
JOIN permissions p ON p.id = rp.permission_id
WHERE p.permission_key LIKE 'contract_template:%'
ORDER BY r.role_key, p.permission_key;
SELECT COUNT(*) AS category_count FROM public.contract_categories;
SELECT COUNT(*) AS template_count FROM public.contract_templates;
SELECT id, template_code, title, file_path, pdf_file_path
FROM public.contract_templates
ORDER BY id
LIMIT 10;
```
#### 当前基线验收结果
- 主库 `leaudit_platform`
- `contract_categories = 9`
- `contract_templates = 27`
- 新 bucket `leaudit`
- `contract-templates/...` 对象总数 = `54`
- 新路径样例
- `contract-templates/买卖合同/mmht/source__买卖合同范本.docx`
- `contract-templates/买卖合同/mmht/preview__买卖合同范本.pdf`
### 七、RAG
- `schema_add_rag_chat.sql`
scripts/创建sql/schema_contract_templates.sql
+66
View File
@@ -0,0 +1,66 @@
BEGIN;
-- ============================================================================
-- LeAudit Platform Contract Template Schema
-- 目标：
-- 1. 在主库 leaudit_platform 创建合同模板分类表
-- 2. 在主库 leaudit_platform 创建合同模板表
-- 3. 补齐索引与基础约束
-- 说明：
-- - 本脚本不依赖旧库 docauditai
-- - 幂等脚本，可重复执行
-- ============================================================================
CREATE TABLE IF NOT EXISTS public.contract_categories (
id SERIAL PRIMARY KEY,
name VARCHAR(100) NOT NULL,
icon VARCHAR(100) NULL,
description TEXT NULL,
sort_order INTEGER NOT NULL DEFAULT 0,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE UNIQUE INDEX IF NOT EXISTS idx_contract_categories_name
ON public.contract_categories(name);
CREATE TABLE IF NOT EXISTS public.contract_templates (
id BIGSERIAL PRIMARY KEY,
template_code VARCHAR(50) NOT NULL,
title VARCHAR(200) NOT NULL,
category_id INTEGER NOT NULL REFERENCES public.contract_categories(id) ON DELETE RESTRICT,
description TEXT NULL,
file_path VARCHAR(500) NULL,
file_format VARCHAR(10) NOT NULL,
is_featured BOOLEAN NOT NULL DEFAULT FALSE,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
pdf_file_path VARCHAR(500) NULL
);
CREATE UNIQUE INDEX IF NOT EXISTS idx_contract_templates_code
ON public.contract_templates(template_code);
CREATE INDEX IF NOT EXISTS idx_contract_templates_category_id
ON public.contract_templates(category_id);
CREATE INDEX IF NOT EXISTS idx_contract_templates_updated_at
ON public.contract_templates(updated_at DESC);
COMMENT ON TABLE public.contract_categories IS '合同模板分类表';
COMMENT ON COLUMN public.contract_categories.name IS '分类名称';
COMMENT ON COLUMN public.contract_categories.icon IS '分类图标';
COMMENT ON COLUMN public.contract_categories.description IS '分类描述';
COMMENT ON COLUMN public.contract_categories.sort_order IS '排序值';
COMMENT ON TABLE public.contract_templates IS '合同模板主表';
COMMENT ON COLUMN public.contract_templates.template_code IS '模板编码';
COMMENT ON COLUMN public.contract_templates.title IS '模板标题';
COMMENT ON COLUMN public.contract_templates.category_id IS '所属分类ID';
COMMENT ON COLUMN public.contract_templates.description IS '模板描述';
COMMENT ON COLUMN public.contract_templates.file_path IS '源模板文件路径';
COMMENT ON COLUMN public.contract_templates.file_format IS '文件格式';
COMMENT ON COLUMN public.contract_templates.is_featured IS '是否推荐模板';
COMMENT ON COLUMN public.contract_templates.pdf_file_path IS 'PDF预览文件路径';
COMMIT;
scripts/创建sql/seed_contract_templates_rbac.sql
+140
View File
@@ -0,0 +1,140 @@
BEGIN;
-- ============================================================================
-- LeAudit Platform Contract Template RBAC Seed
-- 目标：
-- 1. 补齐合同模板读权限
-- 2. 给 super_admin / provincial_admin / admin 分配模板读权限
-- 说明：
-- - 依赖 user_rbac_schema_patch.sql
-- - 依赖合同模板前端路由已存在于 sys_routes
-- - 幂等脚本，可重复执行
-- ============================================================================
WITH route_map AS (
SELECT id, route_path
FROM sys_routes
WHERE deleted_at IS NULL
AND route_path IN ('/contract-template/list', '/contract-template/search')
)
INSERT INTO permissions (
permission_key,
module,
resource,
action,
description,
display_name,
permission_type,
is_system,
metadata,
created_at,
updated_at,
created_by,
updated_by,
parent_id,
sort_order,
route_id,
api_path,
api_method,
related_routes
)
SELECT
seed.permission_key,
seed.module,
seed.resource,
seed.action,
seed.description,
seed.display_name,
'API',
TRUE,
NULL::jsonb,
NOW(),
NOW(),
NULL::bigint,
NULL::bigint,
NULL::bigint,
seed.sort_order,
route_map.id,
seed.api_path,
seed.api_method,
NULL::bigint[]
FROM (
VALUES
('contract_template:list:read', 'contract_template', 'list', 'read', '查看合同模板列表', '查看合同模板列表', '/contract-template/list', 310, '/api/v3/contract-templates', 'GET'),
('contract_template:search:read', 'contract_template', 'search', 'read', '搜索合同模板', '搜索合同模板', '/contract-template/search', 311, '/api/v3/contract-templates/search','GET'),
('contract_template:detail:read', 'contract_template', 'detail', 'read', '查看合同模板详情', '查看合同模板详情', '/contract-template/list', 312, '/api/v3/contract-templates/{id}', 'GET')
) AS seed(
permission_key,
module,
resource,
action,
description,
display_name,
route_path,
sort_order,
api_path,
api_method
)
JOIN route_map ON route_map.route_path = seed.route_path
ON CONFLICT (permission_key) DO UPDATE SET
module = EXCLUDED.module,
resource = EXCLUDED.resource,
action = EXCLUDED.action,
description = EXCLUDED.description,
display_name = EXCLUDED.display_name,
permission_type = EXCLUDED.permission_type,
is_system = EXCLUDED.is_system,
route_id = EXCLUDED.route_id,
api_path = EXCLUDED.api_path,
api_method = EXCLUDED.api_method,
sort_order = EXCLUDED.sort_order,
updated_at = NOW();
WITH role_map AS (
SELECT id, role_key
FROM roles
WHERE role_key IN ('super_admin', 'provincial_admin', 'admin')
),
perm_map AS (
SELECT id, permission_key
FROM permissions
WHERE permission_key LIKE 'contract_template:%'
),
seed(role_key, permission_key, grant_type, data_scope) AS (
VALUES
('super_admin', 'contract_template:list:read', 'GRANT', 'ALL'),
('super_admin', 'contract_template:search:read', 'GRANT', 'ALL'),
('super_admin', 'contract_template:detail:read', 'GRANT', 'ALL'),
('provincial_admin', 'contract_template:list:read', 'GRANT', 'ALL'),
('provincial_admin', 'contract_template:search:read', 'GRANT', 'ALL'),
('provincial_admin', 'contract_template:detail:read', 'GRANT', 'ALL'),
('admin', 'contract_template:list:read', 'GRANT', 'DEPT'),
('admin', 'contract_template:search:read', 'GRANT', 'DEPT'),
('admin', 'contract_template:detail:read', 'GRANT', 'DEPT')
)
INSERT INTO role_permissions (
role_id,
permission_id,
grant_type,
data_scope,
created_at,
updated_at
)
SELECT
role_map.id,
perm_map.id,
seed.grant_type,
seed.data_scope,
NOW(),
NOW()
FROM seed
JOIN role_map ON role_map.role_key = seed.role_key
JOIN perm_map ON perm_map.permission_key = seed.permission_key
ON CONFLICT (role_id, permission_id) DO UPDATE SET
grant_type = EXCLUDED.grant_type,
data_scope = EXCLUDED.data_scope,
updated_at = NOW();
COMMIT;