feat: 完善Dify知识库管理召回测试模块,优化知识库上传文件时的分段配置设置

This commit is contained in:
PingChuan
2025-12-03 12:03:09 +08:00
parent 0f49426a2e
commit a13f3b3635
29 changed files with 4016 additions and 1880 deletions
@@ -1,16 +1,10 @@
import { useState, useEffect } from 'react';
import { Form, Input, Button, Card, message, Spin } from 'antd';
import { Form, Input, Button, Card, Spin } from 'antd';
import { SaveOutlined } from '@ant-design/icons';
import type { Dataset } from '~/api/dify-dataset/type/datasetTypes';
import { updateDatasetName } from '~/api/dify-dataset/api/datasetApi';
import { useDatasetSettings } from '~/hooks/dify-dataset-manager/dataset-settings';
import type { DatasetSettingsProps } from '~/types/dify-dataset-manager/dataset-settings';
const { TextArea } = Input;
interface DatasetSettingsProps {
dataset: Dataset | null;
onDatasetUpdated: (dataset: Dataset) => void;
}
/**
* 知识库设置组件
* 用于修改知识库名称和描述
@@ -20,70 +14,14 @@ export default function DatasetSettings({
onDatasetUpdated,
}: DatasetSettingsProps) {
const [form] = Form.useForm();
const [saving, setSaving] = useState(false);
const [hasChanges, setHasChanges] = useState(false);
// 初始化表单数据
useEffect(() => {
if (dataset) {
form.setFieldsValue({
name: dataset.name,
description: dataset.description || '',
});
setHasChanges(false);
}
}, [dataset, form]);
/**
* 处理表单值变化
*/
const handleValuesChange = () => {
const values = form.getFieldsValue();
const changed =
values.name !== dataset?.name ||
values.description !== (dataset?.description || '');
setHasChanges(changed);
};
/**
* 保存设置
*/
const handleSave = async () => {
if (!dataset) {
message.error('知识库不存在');
return;
}
try {
const values = await form.validateFields();
setSaving(true);
// 目前只支持修改名称
const updatedDataset = await updateDatasetName(dataset.id, values.name);
message.success('保存成功');
onDatasetUpdated(updatedDataset);
setHasChanges(false);
} catch (err: any) {
console.error('保存设置失败:', err);
message.error(err.message || '保存失败');
} finally {
setSaving(false);
}
};
/**
* 重置表单
*/
const handleReset = () => {
if (dataset) {
form.setFieldsValue({
name: dataset.name,
description: dataset.description || '',
});
setHasChanges(false);
}
};
const {
saving,
hasChanges,
handleValuesChange,
handleSave,
handleReset,
} = useDatasetSettings(dataset, form, onDatasetUpdated);
if (!dataset) {
return (
@@ -1,4 +1,3 @@
import { useState, useEffect } from 'react';
import {
Input,
Button,
@@ -8,7 +7,6 @@ import {
Card,
Empty,
Spin,
message,
Divider,
Tooltip,
} from 'antd';
@@ -17,45 +15,8 @@ import {
ReloadOutlined,
EyeOutlined,
} from '@ant-design/icons';
import type { Document } from '~/api/dify-dataset/type/documentTypes';
import type { Segment } from '~/api/dify-dataset/type';
import { fetchSegments } from '~/api/dify-dataset/api/segmentApi';
import { updateDocumentWithSettings } from '~/api/dify-dataset/api/documentApi';
interface DocumentDetailProps {
datasetId: string;
document: Document | null;
}
/**
* 分段设置配置
* 注意:Dify API 支持的参数有限
* - separator: ✅ 支持
* - maxTokens: ✅ 支持
* - removeExtraSpaces: ✅ 支持
* - removeUrlsEmails: ✅ 支持
* - useQASegment: ⚠️ 需要 doc_form: "qa_model"
*/
interface SegmentationSettings {
separator: string;
maxTokens: number;
removeExtraSpaces: boolean;
removeUrlsEmails: boolean;
useQASegment: boolean;
qaLanguage: string;
}
/**
* 默认分段设置
*/
const DEFAULT_SETTINGS: SegmentationSettings = {
separator: '\\n\\n',
maxTokens: 500,
removeExtraSpaces: true,
removeUrlsEmails: false,
useQASegment: false,
qaLanguage: 'Chinese',
};
import { useDocumentDetail } from '~/hooks/dify-dataset-manager/document-detail';
import type { DocumentDetailProps } from '~/types/dify-dataset-manager/document-detail';
/**
* 文档详情组件
@@ -65,98 +26,17 @@ export default function DocumentDetail({
datasetId,
document,
}: DocumentDetailProps) {
// 分段设置状态
const [settings, setSettings] = useState<SegmentationSettings>(DEFAULT_SETTINGS);
// 预览状态
const [previewSegments, setPreviewSegments] = useState<Segment[]>([]);
const [previewLoading, setPreviewLoading] = useState(false);
const [showPreview, setShowPreview] = useState(false);
// 保存状态
const [saving, setSaving] = useState(false);
// 当文档变化时重置设置
useEffect(() => {
if (document) {
// 可以从文档中读取已有的设置,这里使用默认值
setSettings(DEFAULT_SETTINGS);
setPreviewSegments([]);
setShowPreview(false);
}
}, [document?.id]);
/**
* 更新设置
*/
const updateSettings = (key: keyof SegmentationSettings, value: any) => {
setSettings(prev => ({ ...prev, [key]: value }));
};
/**
* 重置设置
*/
const handleReset = () => {
setSettings(DEFAULT_SETTINGS);
setPreviewSegments([]);
setShowPreview(false);
};
/**
* 预览分段
*/
const handlePreview = async () => {
if (!document) return;
setPreviewLoading(true);
setShowPreview(true);
try {
// 获取当前文档的分段作为预览
const response = await fetchSegments(datasetId, document.id, 1, 50);
setPreviewSegments(response.data || []);
if (response.data?.length === 0) {
message.info('该文档暂无分段数据');
}
} catch (err: any) {
console.error('预览分段失败:', err);
message.error(err.message || '预览失败');
} finally {
setPreviewLoading(false);
}
};
/**
* 保存并处理
*/
const handleSaveAndProcess = async () => {
if (!document) return;
setSaving(true);
try {
await updateDocumentWithSettings(datasetId, document.id, {
indexing_technique: 'high_quality',
process_rule: {
mode: 'custom',
rules: {
pre_processing_rules: [
{ id: 'remove_extra_spaces', enabled: settings.removeExtraSpaces },
{ id: 'remove_urls_emails', enabled: settings.removeUrlsEmails },
],
segmentation: {
separator: settings.separator.replace(/\\n/g, '\n'),
max_tokens: settings.maxTokens,
},
},
},
});
message.success('设置已保存,文档正在重新处理...');
} catch (err: any) {
console.error('保存设置失败:', err);
message.error(err.message || '保存失败');
} finally {
setSaving(false);
}
};
const {
settings,
previewSegments,
previewLoading,
showPreview,
saving,
updateSettings,
handleReset,
handlePreview,
handleSaveAndProcess,
} = useDocumentDetail(datasetId, document);
if (!document) {
return (
@@ -1,4 +1,3 @@
import { useState } from 'react';
import {
Button,
Input,
@@ -8,7 +7,6 @@ import {
Tooltip,
Popconfirm,
Switch,
message,
Empty,
Spin,
} from 'antd';
@@ -19,33 +17,14 @@ import {
FileTextOutlined,
CloudUploadOutlined,
EyeOutlined,
ClockCircleOutlined,
CheckCircleOutlined,
SyncOutlined,
ExclamationCircleOutlined,
PauseCircleOutlined,
} from '@ant-design/icons';
import type { ColumnsType } from 'antd/es/table';
import type { Document, IndexingStatus } from '~/api/dify-dataset/type/documentTypes';
import { deleteDocument, toggleDocumentStatus } from '~/api/dify-dataset/api/documentApi';
import { useDocumentList } from '~/hooks/dify-dataset-manager/document-list';
import type { DocumentListProps } from '~/types/dify-dataset-manager/document-list';
import DocumentUpload from './document-upload';
import '../../styles/components/dify-dataset-manager/index.css';
interface DocumentListProps {
datasetId: string;
datasetName: string;
documents: Document[];
loading: boolean;
total: number;
page: number;
pageSize: number;
onPageChange: (page: number) => void;
onDocumentDeleted: (documentId: string) => void;
onDocumentStatusChanged: (documentId: string, enabled: boolean) => void;
onRefresh: () => void;
onViewDocument?: (document: Document) => void;
}
/**
* 文档列表组件
*/
@@ -62,116 +41,24 @@ export default function DocumentList({
onRefresh,
onViewDocument,
}: DocumentListProps) {
const [searchValue, setSearchValue] = useState('');
const [deletingId, setDeletingId] = useState<string | null>(null);
// 显示上传页面的状态
const [showUploadPage, setShowUploadPage] = useState(false);
/**
* 获取状态标签配置
*/
const getStatusConfig = (status: IndexingStatus) => {
const configs: Record<IndexingStatus, { color: string; icon: React.ReactNode; text: string }> = {
completed: { color: 'success', icon: <CheckCircleOutlined />, text: '已完成' },
indexing: { color: 'processing', icon: <SyncOutlined spin />, text: '索引中' },
waiting: { color: 'warning', icon: <ClockCircleOutlined />, text: '等待中' },
parsing: { color: 'processing', icon: <SyncOutlined spin />, text: '解析中' },
cleaning: { color: 'processing', icon: <SyncOutlined spin />, text: '清洗中' },
splitting: { color: 'processing', icon: <SyncOutlined spin />, text: '分段中' },
paused: { color: 'default', icon: <PauseCircleOutlined />, text: '已暂停' },
error: { color: 'error', icon: <ExclamationCircleOutlined />, text: '错误' },
};
return configs[status] || { color: 'default', icon: null, text: status };
};
/**
* 格式化日期
*/
const formatDate = (timestamp: number) => {
return new Date(timestamp * 1000).toLocaleString('zh-CN', {
year: 'numeric',
month: '2-digit',
day: '2-digit',
hour: '2-digit',
minute: '2-digit',
});
};
/**
* 格式化数字
*/
const formatNumber = (num: number) => {
if (num >= 10000) {
return (num / 10000).toFixed(1) + 'w';
}
if (num >= 1000) {
return (num / 1000).toFixed(1) + 'k';
}
return num.toString();
};
/**
* 处理删除文档
*/
const handleDelete = async (documentId: string) => {
setDeletingId(documentId);
try {
await deleteDocument(datasetId, documentId);
message.success('删除成功');
onDocumentDeleted(documentId);
} catch (err: any) {
console.error('删除文档失败:', err);
message.error(err.message || '删除失败');
} finally {
setDeletingId(null);
}
};
/**
* 处理启用/禁用文档
*/
const handleToggleStatus = async (documentId: string, enabled: boolean) => {
try {
await toggleDocumentStatus(datasetId, documentId, enabled);
message.success(enabled ? '已启用' : '已禁用');
onDocumentStatusChanged(documentId, enabled);
} catch (err: any) {
console.error('切换文档状态失败:', err);
message.error(err.message || '操作失败');
}
};
/**
* 点击上传按钮,显示上传页面
*/
const handleUploadClick = () => {
if (!datasetId) {
message.error('请先选择知识库');
return;
}
setShowUploadPage(true);
};
/**
* 关闭上传页面
*/
const handleUploadClose = () => {
setShowUploadPage(false);
};
/**
* 上传成功回调
*/
const handleUploadSuccess = () => {
setShowUploadPage(false);
onRefresh();
};
const {
searchValue,
setSearchValue,
deletingId,
showUploadPage,
getStatusConfig,
formatDate,
formatNumber,
handleDelete,
handleToggleStatus,
handleUploadClick,
handleUploadClose,
handleUploadSuccess,
filterDocuments,
} = useDocumentList(datasetId, onDocumentDeleted, onDocumentStatusChanged, onRefresh);
// 过滤文档
const filteredDocuments = documents.filter((doc) =>
doc.name.toLowerCase().includes(searchValue.toLowerCase())
);
const filteredDocuments = filterDocuments(documents);
// 表格列定义
const columns: ColumnsType<Document> = [
@@ -1,110 +1,36 @@
import { useState, useEffect, useRef, useCallback } from 'react';
import {
Input,
ArrowLeftOutlined,
CheckCircleOutlined,
DeleteOutlined,
ExclamationCircleOutlined,
FileTextOutlined,
InboxOutlined,
LoadingOutlined,
QuestionCircleOutlined,
} from '@ant-design/icons';
import type { UploadFile } from 'antd';
import {
Button,
InputNumber,
Checkbox,
Select,
Card,
Empty,
Spin,
message,
Checkbox,
Divider,
Tooltip,
Empty,
Input,
InputNumber,
Progress,
Select,
Spin,
Tooltip,
Upload,
} from 'antd';
import type { UploadFile, UploadProps } from 'antd';
import {
QuestionCircleOutlined,
FileTextOutlined,
CheckCircleOutlined,
LoadingOutlined,
ExclamationCircleOutlined,
InboxOutlined,
DeleteOutlined,
ArrowLeftOutlined,
} from '@ant-design/icons';
import type { IndexingStatus } from '~/api/dify-dataset/type/documentTypes';
import { useEffect, useState } from 'react';
import type { Segment } from '~/api/dify-dataset/type';
import {
uploadDocumentWithConfig,
updateDocumentByFile,
fetchIndexingStatus,
} from '~/api/dify-dataset/api/documentApi';
import { fetchSegments } from '~/api/dify-dataset/api/segmentApi';
import { useDocumentUpload } from '~/hooks/dify-dataset-manager/document-upload';
import type { DocumentUploadProps, UploadedDocument } from '~/types/dify-dataset-manager/document-upload';
import { SUPPORTED_FORMATS } from '~/types/dify-dataset-manager/document-upload';
const { Dragger } = Upload;
interface DocumentUploadProps {
datasetId: string;
onClose: () => void;
onSuccess: () => void;
}
/**
* 分段设置配置
*/
interface SegmentationSettings {
separator: string;
maxTokens: number;
chunkOverlap: number;
removeExtraSpaces: boolean;
removeUrlsEmails: boolean;
indexingTechnique: 'high_quality' | 'economy';
}
/**
* 默认分段设置
*/
const DEFAULT_SETTINGS: SegmentationSettings = {
separator: '\\n\\n',
maxTokens: 1024,
chunkOverlap: 50,
removeExtraSpaces: true,
removeUrlsEmails: false,
indexingTechnique: 'high_quality',
};
/**
* 单个文档的上传状态
*/
type DocumentStage = 'pending' | 'uploading' | 'indexing' | 'completed' | 'error';
/**
* 上传的文档信息(支持多文件)
*/
interface UploadedDocument {
file: File;
documentId: string;
batch: string;
stage: DocumentStage;
indexingStatus: IndexingStatus;
uploadProgress: number;
error?: string;
settings: SegmentationSettings;
segments: Segment[];
}
/**
* 索引状态配置
*/
const INDEXING_STATUS_CONFIG: Record<IndexingStatus, { text: string; percent: number }> = {
waiting: { text: '等待处理...', percent: 10 },
parsing: { text: '解析文档...', percent: 30 },
cleaning: { text: '清洗文本...', percent: 50 },
splitting: { text: '分段处理...', percent: 70 },
indexing: { text: '建立索引...', percent: 85 },
completed: { text: '处理完成', percent: 100 },
paused: { text: '已暂停', percent: 0 },
error: { text: '处理失败', percent: 0 },
};
/**
* 支持的文件格式
*/
const SUPPORTED_FORMATS = 'TXT, MARKDOWN, MDX, PDF, HTML, XLSX, XLS, DOCX, CSV, VTT, PROPERTIES, MD, HTM';
/**
* 文档上传组件
* 支持多文件上传,两步流程:选择文件 → 上传并配置分段
@@ -114,419 +40,55 @@ export default function DocumentUpload({
onClose,
onSuccess,
}: DocumentUploadProps) {
// 步骤控制
const [step, setStep] = useState<1 | 2>(1);
const {
// 状态
step,
fileList,
uploadedDocuments,
currentSettings,
previewLoading,
// 文件相关
const [selectedFiles, setSelectedFiles] = useState<File[]>([]);
const [fileList, setFileList] = useState<UploadFile[]>([]);
// 方法
handleFileChange,
handleRemoveFile,
handleNextStep,
handleDocumentChange,
handleReprocess,
handlePrevStep,
handleGoToDocuments,
updateCurrentSettings,
// 多文档状态管理
const [uploadedDocuments, setUploadedDocuments] = useState<UploadedDocument[]>([]);
// 当前选中查看的文档索引
const [currentDocIndex, setCurrentDocIndex] = useState(0);
// 计算属性方法
getCurrentDocument,
getCurrentProgress,
getStatusText,
isCurrentDocProcessing,
getCompletionStats,
} = useDocumentUpload(datasetId, onClose, onSuccess);
// 当前显示的分段设置(来自当前选中的文档)
const [currentSettings, setCurrentSettings] = useState<SegmentationSettings>(DEFAULT_SETTINGS);
const selectedFiles = fileList.filter((f: UploadFile) => f.originFileObj).map((f: UploadFile) => f.originFileObj as File);
// 预览相关
const [previewLoading, setPreviewLoading] = useState(false);
// 平滑进度条逻辑
const [displayPercent, setDisplayPercent] = useState(0);
const targetPercent = getCurrentProgress();
// 轮询定时器(支持多个文档)
const pollingTimersRef = useRef<Map<string, NodeJS.Timeout>>(new Map());
// 清理所有轮询定时器
useEffect(() => {
return () => {
pollingTimersRef.current.forEach(timer => clearInterval(timer));
pollingTimersRef.current.clear();
};
}, []);
if (targetPercent > displayPercent) {
// 如果目标进度大于当前显示进度,启动动画
const diff = targetPercent - displayPercent;
// 动态步长:差距越大跑得越快,但最小步长为1
const step = Math.max(1, Math.ceil(diff / 10));
/**
* 停止指定文档的轮询
*/
const stopPolling = useCallback((documentId: string) => {
const timer = pollingTimersRef.current.get(documentId);
if (timer) {
clearInterval(timer);
pollingTimersRef.current.delete(documentId);
const timer = requestAnimationFrame(() => {
setDisplayPercent(prev => Math.min(targetPercent, prev + step));
});
return () => cancelAnimationFrame(timer);
} else if (targetPercent < displayPercent && targetPercent === 0) {
// 如果目标重置为0(例如重新开始),立即重置
setDisplayPercent(0);
}
}, []);
/**
* 停止所有轮询
*/
const stopAllPolling = useCallback(() => {
pollingTimersRef.current.forEach(timer => clearInterval(timer));
pollingTimersRef.current.clear();
}, []);
/**
* 加载分段预览
*/
const loadSegmentsPreview = useCallback(async (documentId: string, docIndex: number) => {
setPreviewLoading(true);
try {
const response = await fetchSegments(datasetId, documentId, 1, 50);
const segments = response.data || [];
// 更新对应文档的分段
setUploadedDocuments(prev => prev.map((doc, idx) =>
idx === docIndex ? { ...doc, segments } : doc
));
} catch (err: any) {
console.error('加载分段预览失败:', err);
message.error('加载分段预览失败');
} finally {
setPreviewLoading(false);
}
}, [datasetId]);
/**
* 轮询索引状态
*/
const pollIndexingStatus = useCallback(async (batch: string, documentId: string, docIndex: number) => {
try {
const response = await fetchIndexingStatus(datasetId, batch);
const documentStatus = response.data?.[0];
if (documentStatus) {
const status = documentStatus.indexing_status as IndexingStatus;
// 更新文档状态
setUploadedDocuments(prev => prev.map((doc, idx) => {
if (idx !== docIndex) return doc;
return { ...doc, indexingStatus: status };
}));
if (status === 'completed') {
stopPolling(documentId);
setUploadedDocuments(prev => prev.map((doc, idx) =>
idx === docIndex ? { ...doc, stage: 'completed' } : doc
));
// message.success(`文档 "${uploadedDocuments[docIndex]?.file.name}" 处理完成!`);
// 自动加载分段预览
loadSegmentsPreview(documentId, docIndex);
} else if (status === 'error') {
stopPolling(documentId);
setUploadedDocuments(prev => prev.map((doc, idx) =>
idx === docIndex ? { ...doc, stage: 'error', error: documentStatus.error || '处理失败' } : doc
));
}
}
} catch (err) {
console.error('获取索引状态失败:', err);
}
}, [datasetId, stopPolling, loadSegmentsPreview, uploadedDocuments]);
/**
* 开始轮询
*/
const startPolling = useCallback((batch: string, documentId: string, docIndex: number) => {
// 先停止之前的轮询
stopPolling(documentId);
// 开始新的轮询
const timer = setInterval(() => {
pollIndexingStatus(batch, documentId, docIndex);
}, 2000);
pollingTimersRef.current.set(documentId, timer);
// 立即执行一次
pollIndexingStatus(batch, documentId, docIndex);
}, [stopPolling, pollIndexingStatus]);
/**
* 构建上传配置
*/
const buildConfig = (s: SegmentationSettings) => ({
indexing_technique: s.indexingTechnique,
process_rule: {
mode: 'custom' as const,
rules: {
pre_processing_rules: [
{ id: 'remove_extra_spaces' as const, enabled: s.removeExtraSpaces },
{ id: 'remove_urls_emails' as const, enabled: s.removeUrlsEmails },
],
segmentation: {
separator: s.separator.replace(/\\n/g, '\n'),
max_tokens: s.maxTokens,
},
},
},
});
/**
* 更新当前文档的设置
*/
const updateCurrentSettings = (key: keyof SegmentationSettings, value: any) => {
const newSettings = { ...currentSettings, [key]: value };
setCurrentSettings(newSettings);
// 同步更新到文档列表
setUploadedDocuments(prev => prev.map((doc, idx) =>
idx === currentDocIndex ? { ...doc, settings: newSettings } : doc
));
};
/**
* 处理文件选择变化
*/
const handleFileChange: UploadProps['onChange'] = ({ fileList: newFileList }) => {
setFileList(newFileList);
// 提取实际文件对象
const files = newFileList
.filter(f => f.originFileObj)
.map(f => f.originFileObj as File);
setSelectedFiles(files);
};
/**
* 移除文件
*/
const handleRemoveFile = (file: UploadFile) => {
const newFileList = fileList.filter(f => f.uid !== file.uid);
setFileList(newFileList);
const files = newFileList
.filter(f => f.originFileObj)
.map(f => f.originFileObj as File);
setSelectedFiles(files);
};
/**
* 上传单个文件
*/
const uploadSingleFile = async (file: File, index: number): Promise<void> => {
try {
// 更新状态为上传中
setUploadedDocuments(prev => prev.map((doc, idx) =>
idx === index ? { ...doc, stage: 'uploading' as DocumentStage } : doc
));
const config = buildConfig(DEFAULT_SETTINGS);
const result = await uploadDocumentWithConfig(
datasetId,
file,
config,
(percent) => {
setUploadedDocuments(prev => prev.map((doc, idx) =>
idx === index ? { ...doc, uploadProgress: percent } : doc
));
}
);
// 更新文档信息
setUploadedDocuments(prev => prev.map((doc, idx) =>
idx === index ? {
...doc,
documentId: result.document.id,
batch: result.batch,
stage: 'indexing' as DocumentStage,
indexingStatus: 'waiting' as IndexingStatus,
} : doc
));
// 开始轮询索引状态
startPolling(result.batch, result.document.id, index);
} catch (err: any) {
console.error(`上传文档 ${file.name} 失败:`, err);
setUploadedDocuments(prev => prev.map((doc, idx) =>
idx === index ? {
...doc,
stage: 'error' as DocumentStage,
error: err.message || '上传失败',
} : doc
));
}
};
/**
* 点击"下一步":立即上传所有文件
*/
const handleNextStep = async () => {
if (selectedFiles.length === 0) {
message.warning('请先选择文件');
return;
}
// 初始化所有文档状态
const docs: UploadedDocument[] = selectedFiles.map(file => ({
file,
documentId: '',
batch: '',
stage: 'pending' as DocumentStage,
indexingStatus: 'waiting' as IndexingStatus,
uploadProgress: 0,
settings: { ...DEFAULT_SETTINGS },
segments: [],
}));
setUploadedDocuments(docs);
setCurrentDocIndex(0);
setCurrentSettings({ ...DEFAULT_SETTINGS });
setStep(2);
// 依次上传所有文件
for (let i = 0; i < selectedFiles.length; i++) {
await uploadSingleFile(selectedFiles[i], i);
}
};
/**
* 切换查看的文档
*/
const handleDocumentChange = (docId: string) => {
const index = uploadedDocuments.findIndex(doc => doc.documentId === docId || doc.file.name === docId);
if (index !== -1) {
setCurrentDocIndex(index);
const doc = uploadedDocuments[index];
setCurrentSettings(doc.settings);
}
};
/**
* 修改参数后重新处理当前文档
*/
const handleReprocess = async () => {
const currentDoc = uploadedDocuments[currentDocIndex];
if (!currentDoc || !currentDoc.documentId) return;
// 更新状态
setUploadedDocuments(prev => prev.map((doc, idx) =>
idx === currentDocIndex ? {
...doc,
stage: 'uploading' as DocumentStage,
uploadProgress: 0,
segments: [],
} : doc
));
try {
const config = buildConfig(currentSettings);
const result = await updateDocumentByFile(
datasetId,
currentDoc.documentId,
currentDoc.file,
config,
(percent) => {
setUploadedDocuments(prev => prev.map((doc, idx) =>
idx === currentDocIndex ? { ...doc, uploadProgress: percent } : doc
));
}
);
// 更新 batch
setUploadedDocuments(prev => prev.map((doc, idx) =>
idx === currentDocIndex ? {
...doc,
batch: result.batch,
stage: 'indexing' as DocumentStage,
indexingStatus: 'waiting' as IndexingStatus,
} : doc
));
startPolling(result.batch, currentDoc.documentId, currentDocIndex);
} catch (err: any) {
console.error('重新处理失败:', err);
setUploadedDocuments(prev => prev.map((doc, idx) =>
idx === currentDocIndex ? {
...doc,
stage: 'error' as DocumentStage,
error: err.message || '重新处理失败',
} : doc
));
message.error(err.message || '重新处理失败');
}
};
/**
* 返回上一步
*/
const handlePrevStep = () => {
// 检查是否有文档正在处理
const hasProcessing = uploadedDocuments.some(doc =>
doc.stage === 'uploading' || doc.stage === 'indexing'
);
if (hasProcessing) {
message.warning('还有文档正在处理中,请等待完成');
return;
}
stopAllPolling();
setStep(1);
setUploadedDocuments([]);
setCurrentDocIndex(0);
setCurrentSettings(DEFAULT_SETTINGS);
};
/**
* 返回文档列表
*/
const handleGoToDocuments = () => {
stopAllPolling();
const hasCompleted = uploadedDocuments.some(doc => doc.stage === 'completed');
if (hasCompleted) {
onSuccess();
}
onClose();
};
/**
* 获取当前文档
*/
const getCurrentDocument = (): UploadedDocument | null => {
return uploadedDocuments[currentDocIndex] || null;
};
/**
* 获取当前文档的进度
*/
const getCurrentProgress = () => {
const doc = getCurrentDocument();
if (!doc) return 0;
if (doc.stage === 'uploading') {
return doc.uploadProgress;
}
if (doc.stage === 'indexing' || doc.stage === 'completed') {
return INDEXING_STATUS_CONFIG[doc.indexingStatus]?.percent || 0;
}
return 0;
};
/**
* 获取当前文档的状态文本
*/
const getStatusText = () => {
const doc = getCurrentDocument();
if (!doc) return '';
if (doc.stage === 'uploading') {
return `正在上传... ${doc.uploadProgress}%`;
}
if (doc.stage === 'indexing') {
return INDEXING_STATUS_CONFIG[doc.indexingStatus]?.text || '处理中...';
}
if (doc.stage === 'completed') {
return '处理完成';
}
if (doc.stage === 'error') {
return doc.error || '处理失败';
}
return '';
};
/**
* 判断当前文档是否正在处理
*/
const isCurrentDocProcessing = () => {
const doc = getCurrentDocument();
return doc?.stage === 'uploading' || doc?.stage === 'indexing';
};
/**
* 获取所有文档的完成状态统计
*/
const getCompletionStats = () => {
const completed = uploadedDocuments.filter(doc => doc.stage === 'completed').length;
const total = uploadedDocuments.length;
return { completed, total };
};
}, [targetPercent, displayPercent]);
/**
* 渲染步骤指示器(两步流程)
@@ -567,7 +129,7 @@ export default function DocumentUpload({
<p className="ant-upload-drag-icon">
<InboxOutlined />
</p>
<p className="ant-upload-text"> <span className="upload-link"></span></p>
<p className="ant-upload-text"> <span className="upload-link"></span></p>
<p className="ant-upload-hint">
{SUPPORTED_FORMATS} 15MB
</p>
@@ -579,7 +141,7 @@ export default function DocumentUpload({
<div className="selected-files-section">
<h3 className="section-subtitle"> ({selectedFiles.length} )</h3>
<div className="selected-files-list">
{fileList.map((file) => (
{fileList.map((file: UploadFile) => (
<div key={file.uid} className="selected-file-item">
<FileTextOutlined className="file-icon" />
<div className="file-info">
@@ -768,7 +330,7 @@ export default function DocumentUpload({
value={currentDoc?.documentId || currentDoc?.file.name}
style={{ width: 500 }}
onChange={handleDocumentChange}
options={uploadedDocuments.map((doc, idx) => ({
options={uploadedDocuments.map((doc: UploadedDocument) => ({
value: doc.documentId || doc.file.name,
label: (
<span className="file-select-option">
@@ -800,7 +362,7 @@ export default function DocumentUpload({
<LoadingOutlined className="status-icon loading" />
</div>
<Progress
percent={getCurrentProgress()}
percent={displayPercent}
status="active"
strokeColor={{
'0%': '#00684a',
@@ -808,36 +370,6 @@ export default function DocumentUpload({
}}
/>
<div className="status-text">{getStatusText()}</div>
{/* 索引阶段详情 */}
{currentDoc?.stage === 'indexing' && (
<div className="indexing-stages">
<div className={`stage-item ${['waiting', 'parsing', 'cleaning', 'splitting', 'indexing', 'completed'].includes(currentDoc.indexingStatus) ? 'active' : ''}`}>
<span className="stage-dot"></span>
<span></span>
</div>
<div className={`stage-item ${['parsing', 'cleaning', 'splitting', 'indexing', 'completed'].includes(currentDoc.indexingStatus) ? 'active' : ''}`}>
<span className="stage-dot"></span>
<span></span>
</div>
<div className={`stage-item ${['cleaning', 'splitting', 'indexing', 'completed'].includes(currentDoc.indexingStatus) ? 'active' : ''}`}>
<span className="stage-dot"></span>
<span></span>
</div>
<div className={`stage-item ${['splitting', 'indexing', 'completed'].includes(currentDoc.indexingStatus) ? 'active' : ''}`}>
<span className="stage-dot"></span>
<span></span>
</div>
<div className={`stage-item ${['indexing', 'completed'].includes(currentDoc.indexingStatus) ? 'active' : ''}`}>
<span className="stage-dot"></span>
<span></span>
</div>
<div className={`stage-item ${currentDoc.indexingStatus === 'completed' ? 'active' : ''}`}>
<span className="stage-dot"></span>
<span></span>
</div>
</div>
)}
</div>
) : currentDoc?.stage === 'error' ? (
<div className="preview-error">
@@ -855,7 +387,7 @@ export default function DocumentUpload({
</div>
) : (
<div className="preview-segments">
{currentDoc?.segments.map((segment, index) => (
{currentDoc?.segments.map((segment: Segment, index: number) => (
<div key={segment.id} className="segment-item">
<div className="segment-header">
<span className="segment-index">#{index + 1}</span>
@@ -881,7 +413,7 @@ export default function DocumentUpload({
{stats.completed}/{stats.total}
</span>
<Button type="primary" onClick={handleGoToDocuments}>
</Button>
</div>
)}
+27 -163
View File
@@ -1,14 +1,10 @@
import { useEffect, useState } from 'react';
import { message, Spin } from 'antd';
import DatasetLayout, { type MenuTab } from './layout';
import { Spin } from 'antd';
import DatasetLayout from './layout';
import DocumentList from './document-list';
import DocumentDetail from './document-detail';
import RetrieveTest from './retrieve-test';
import DatasetSettings from './dataset-settings';
import type { Dataset } from '~/api/dify-dataset/type/datasetTypes';
import type { Document } from '~/api/dify-dataset/type/documentTypes';
import { fetchDatasets } from '~/api/dify-dataset/api/datasetApi';
import { fetchDocuments } from '~/api/dify-dataset/api/documentApi';
import { useDatasetManager } from '~/hooks/dify-dataset-manager';
import '../../styles/components/dify-dataset-manager/index.css';
/**
@@ -16,162 +12,30 @@ import '../../styles/components/dify-dataset-manager/index.css';
* 带左侧菜单栏的完整布局
*/
export default function DatasetManager() {
// 知识库状态
const [dataset, setDataset] = useState<Dataset | null>(null);
const [loadingDataset, setLoadingDataset] = useState(true);
// 文档状态
const [documents, setDocuments] = useState<Document[]>([]);
const [loadingDocuments, setLoadingDocuments] = useState(false);
const [documentTotal, setDocumentTotal] = useState(0);
const [documentPage, setDocumentPage] = useState(1);
const [documentPageSize] = useState(20);
// 初始化状态
const [inited, setInited] = useState(false);
const [error, setError] = useState<string | null>(null);
// 菜单状态
const [activeTab, setActiveTab] = useState<MenuTab>('documents');
// 选中的文档(用于查看文档详情)
const [selectedDocument, setSelectedDocument] = useState<Document | null>(null);
/**
* 加载知识库(获取第一个知识库)
*/
const loadDataset = async () => {
setLoadingDataset(true);
try {
console.log('[DatasetManager] 加载知识库...');
const response = await fetchDatasets(1, 1);
console.log('[DatasetManager] 知识库响应:', response);
if (response && response.data && response.data.length > 0) {
const firstDataset = response.data[0];
setDataset(firstDataset);
// 立即加载文档
await loadDocuments(firstDataset.id, 1);
} else {
setError('未找到知识库,请先在Dify中创建知识库');
}
} catch (err: any) {
console.error('[DatasetManager] 加载知识库失败:', err);
setError(err.message || '加载知识库失败');
message.error('加载知识库失败');
} finally {
setLoadingDataset(false);
setInited(true);
}
};
/**
* 加载文档列表
*/
const loadDocuments = async (datasetId: string, page: number = 1) => {
if (!datasetId) return;
setLoadingDocuments(true);
try {
console.log('[DatasetManager] 加载文档列表:', { datasetId, page });
const response = await fetchDocuments(datasetId, page, documentPageSize);
console.log('[DatasetManager] 文档列表响应:', response);
if (response && response.data) {
setDocuments(response.data);
setDocumentTotal(response.total);
setDocumentPage(page);
}
} catch (err: any) {
console.error('[DatasetManager] 加载文档列表失败:', err);
message.error('加载文档列表失败');
} finally {
setLoadingDocuments(false);
}
};
/**
* 处理文档页码变化
*/
const handlePageChange = (page: number) => {
if (dataset) {
loadDocuments(dataset.id, page);
}
};
/**
* 处理文档删除
*/
const handleDocumentDeleted = (documentId: string) => {
setDocuments((prev) => prev.filter((doc) => doc.id !== documentId));
setDocumentTotal((prev) => prev - 1);
// 更新知识库的文档数量
if (dataset) {
setDataset({
...dataset,
document_count: dataset.document_count - 1
});
}
};
/**
* 处理文档状态变化
*/
const handleDocumentStatusChanged = (documentId: string, enabled: boolean) => {
setDocuments((prev) =>
prev.map((doc) =>
doc.id === documentId ? { ...doc, enabled } : doc
)
);
};
/**
* 刷新文档列表
*/
const handleRefresh = () => {
if (dataset) {
loadDocuments(dataset.id, documentPage);
}
};
/**
* 查看文档详情(分段管理)
*/
const handleViewDocument = (doc: Document) => {
console.log('[DatasetManager] 查看文档详情:', doc);
setSelectedDocument(doc);
};
/**
* 返回文档列表
*/
const handleBackToDocuments = () => {
setSelectedDocument(null);
};
/**
* 处理菜单切换
*/
const handleTabChange = (tab: MenuTab) => {
setActiveTab(tab);
// 切换菜单时清除选中的文档
if (tab !== 'documents') {
setSelectedDocument(null);
}
};
/**
* 处理知识库更新
*/
const handleDatasetUpdated = (updatedDataset: Dataset) => {
setDataset(updatedDataset);
};
// 初始化
useEffect(() => {
loadDataset();
}, []);
const {
// 状态
dataset,
loadingDataset,
documents,
loadingDocuments,
documentTotal,
documentPage,
documentPageSize,
inited,
error,
activeTab,
selectedDocument,
// 方法
handlePageChange,
handleDocumentDeleted,
handleDocumentStatusChanged,
handleRefresh,
handleViewDocument,
handleBackToDocuments,
handleTabChange,
handleDatasetUpdated,
} = useDatasetManager();
// 加载中状态
if (!inited || loadingDataset) {
+5 -23
View File
@@ -1,4 +1,3 @@
import { ReactNode } from 'react';
import { Button, Tooltip } from 'antd';
import {
FileTextOutlined,
@@ -7,27 +6,7 @@ import {
ArrowLeftOutlined,
DatabaseOutlined,
} from '@ant-design/icons';
import type { Dataset } from '~/api/dify-dataset/type/datasetTypes';
/**
* 菜单项类型
*/
export type MenuTab = 'documents' | 'retrieve' | 'settings';
interface DatasetLayoutProps {
/** 知识库信息 */
dataset: Dataset | null;
/** 当前激活的菜单 */
activeTab: MenuTab;
/** 菜单切换回调 */
onTabChange: (tab: MenuTab) => void;
/** 是否显示返回按钮(在文档详情页时显示) */
showBackButton?: boolean;
/** 返回按钮点击回调 */
onBack?: () => void;
/** 子组件 */
children: ReactNode;
}
import type { DatasetLayoutProps, MenuTab, MenuItem } from '~/types/dify-dataset-manager/layout';
/**
* 知识库布局组件
@@ -41,7 +20,7 @@ export default function DatasetLayout({
onBack,
children,
}: DatasetLayoutProps) {
const menuItems: { key: MenuTab; icon: ReactNode; label: string }[] = [
const menuItems: MenuItem[] = [
{ key: 'documents', icon: <FileTextOutlined />, label: '文档' },
{ key: 'retrieve', icon: <SearchOutlined />, label: '召回测试' },
{ key: 'settings', icon: <SettingOutlined />, label: '设置' },
@@ -106,3 +85,6 @@ export default function DatasetLayout({
</div>
);
}
// 重新导出类型,保持向后兼容
export type { MenuTab } from '~/types/dify-dataset-manager/layout';
@@ -1,202 +1,306 @@
import { useState } from 'react';
import {
Input,
Button,
Card,
Select,
Slider,
Table,
Tag,
Empty,
Spin,
message,
} from 'antd';
import { FileSearchOutlined } from '@ant-design/icons';
import type { ColumnsType } from 'antd/es/table';
import { SearchOutlined, FileSearchOutlined } from '@ant-design/icons';
import { Button, Tag, Input, Slider, Spin, Select, Flex } from 'antd';
import type { RetrieveRecord } from '~/api/dify-dataset/type';
import { retrieveDataset } from '~/api/dify-dataset/api/segmentApi';
import { useRetrieveTest } from '~/hooks/dify-dataset-manager/retrieve-test';
import type { RetrieveTestProps } from '~/types/dify-dataset-manager/retrieve-test';
interface RetrieveTestProps {
datasetId: string;
// 颜色常量
const colors = {
bgContainer: '#fff',
bgLayout: '#f5f5f5',
bgElevated: '#fafafa',
border: '#e8e8e8',
text: '#262626',
textSecondary: '#8c8c8c',
textTertiary: '#bfbfbf',
textQuaternary: '#d9d9d9',
fillTertiary: '#f0f0f0',
};
/**
* 检索结果项组件
*/
function ResultItem({ record, index }: { record: RetrieveRecord; index: number }) {
const scorePercent = (record.score * 100).toFixed(1);
const scoreColor = record.score > 0.8 ? '#52c41a' : record.score > 0.5 ? '#faad14' : '#666';
return (
<Flex
vertical
gap={12}
style={{
padding: 16,
background: colors.bgContainer,
borderRadius: 8,
border: `1px solid ${colors.border}`,
}}
>
<Flex justify="space-between" align="center">
<Flex gap={8} align="center">
<Tag style={{ background: scoreColor, color: '#fff', border: 'none' }}>
{scorePercent}%
</Tag>
<span style={{ color: colors.textSecondary, fontSize: 12 }}>
#{index + 1} · {record.segment.word_count} · {record.segment.hit_count}
</span>
</Flex>
{record.segment.document && (
<span style={{ color: colors.textTertiary, fontSize: 12 }}>
: {record.segment.document.name}
</span>
)}
</Flex>
<div style={{
color: colors.text,
fontSize: 14,
lineHeight: 1.6,
whiteSpace: 'pre-wrap',
}}>
{record.segment.content.length > 500
? record.segment.content.substring(0, 500) + '...'
: record.segment.content}
</div>
{record.segment.answer && (
<Flex
vertical
gap={4}
style={{
padding: 12,
background: colors.fillTertiary,
borderRadius: 6,
}}
>
<span style={{ color: colors.textSecondary, fontSize: 12 }}>
:
</span>
<span style={{ color: colors.text, fontSize: 14 }}>
{record.segment.answer.length > 200
? record.segment.answer.substring(0, 200) + '...'
: record.segment.answer}
</span>
</Flex>
)}
</Flex>
);
}
/**
* 召回测试组件
* 用于测试知识库的检索效果
*/
export default function RetrieveTest({ datasetId }: RetrieveTestProps) {
const [searchQuery, setSearchQuery] = useState('');
const [retrieveResults, setRetrieveResults] = useState<RetrieveRecord[]>([]);
const [retrieving, setRetrieving] = useState(false);
const [searchMethod, setSearchMethod] = useState<string>('hybrid_search');
const [topK, setTopK] = useState<number>(5);
const {
searchQuery,
setSearchQuery,
retrieveResults,
retrieving,
searchMethod,
setSearchMethod,
topK,
setTopK,
handleRetrieve,
} = useRetrieveTest(datasetId);
/**
* 执行检索
*/
const handleRetrieve = async () => {
if (!searchQuery.trim()) {
message.warning('请输入检索关键词');
return;
}
if (!datasetId) {
message.warning('知识库ID不存在');
return;
}
setRetrieving(true);
try {
const response = await retrieveDataset(datasetId, searchQuery, {
search_method: searchMethod as any,
top_k: topK,
});
setRetrieveResults(response.records || []);
if (response.records?.length === 0) {
message.info('未找到匹配的结果');
}
} catch (err: any) {
console.error('检索失败:', err);
message.error(err.message || '检索失败');
} finally {
setRetrieving(false);
}
};
// 检索结果列定义
const columns: ColumnsType<RetrieveRecord> = [
{
title: '相关度',
dataIndex: 'score',
key: 'score',
width: 100,
render: (score: number) => (
<Tag color={score > 0.8 ? 'green' : score > 0.5 ? 'orange' : 'default'}>
{(score * 100).toFixed(1)}%
</Tag>
),
},
{
title: '内容',
key: 'content',
render: (_, record) => (
<div className="retrieve-result-content">
<div className="content-text">
{record.segment.content.length > 300
? record.segment.content.substring(0, 300) + '...'
: record.segment.content}
</div>
{record.segment.answer && (
<div className="answer-text">
<strong></strong>
{record.segment.answer.length > 150
? record.segment.answer.substring(0, 150) + '...'
: record.segment.answer}
</div>
)}
</div>
),
},
{
title: '字数',
key: 'word_count',
width: 80,
render: (_, record) => record.segment.word_count,
},
{
title: '命中次数',
key: 'hit_count',
width: 100,
render: (_, record) => record.segment.hit_count,
},
// 检索方式选项(只有3种)
const searchMethodOptions = [
{ label: '向量检索', value: 'semantic_search' },
{ label: '全文检索', value: 'full_text_search' },
{ label: '混合检索', value: 'hybrid_search' },
];
return (
<div className="retrieve-test-page">
{/* 页面标题 */}
<div className="page-header">
<h1></h1>
<p className="page-description">
</p>
</div>
<Flex
style={{
height: '100%',
minHeight: 'calc(100vh - 120px)',
}}
>
{/* 左侧面板 - 输入区域 */}
<Flex
vertical
gap={16}
style={{
width: 400,
minWidth: 400,
padding: 20,
background: colors.bgLayout,
borderRight: `1px solid ${colors.border}`,
}}
>
{/* 标题 */}
<Flex vertical gap={4}>
<h2 style={{
margin: 0,
fontSize: 18,
fontWeight: 600,
color: colors.text,
}}>
</h2>
<span style={{
fontSize: 13,
color: colors.textSecondary,
}}>
</span>
</Flex>
{/* 检索设置 */}
<Card className="retrieve-settings" size="small">
<div className="search-row">
<Input
placeholder="输入检索关键词..."
prefix={<FileSearchOutlined />}
value={searchQuery}
onChange={(e) => setSearchQuery(e.target.value)}
onPressEnter={handleRetrieve}
className="search-input"
/>
<Button
type="primary"
onClick={handleRetrieve}
loading={retrieving}
>
</Button>
</div>
<div className="options-row">
<div className="option-item">
<span className="option-label"></span>
{/* 查询输入区 */}
<Flex vertical gap={8}>
<Flex justify="space-between" align="center">
<span style={{
fontSize: 13,
color: colors.text,
fontWeight: 500,
}}>
</span>
<Select
value={searchMethod}
onChange={setSearchMethod}
style={{ width: 140 }}
options={[
{ value: 'keyword_search', label: '关键词搜索' },
{ value: 'semantic_search', label: '语义搜索' },
{ value: 'full_text_search', label: '全文搜索' },
{ value: 'hybrid_search', label: '混合搜索' },
]}
onChange={(value) => setSearchMethod(value as any)}
options={searchMethodOptions}
style={{ width: 130 }}
size="small"
/>
</div>
<div className="option-item">
<span className="option-label"> (Top K)</span>
</Flex>
<Input.TextArea
placeholder="请输入文本,建议使用简短的陈述句。"
value={searchQuery}
onChange={(e) => setSearchQuery(e.target.value)}
onPressEnter={(e) => {
if (!e.shiftKey) {
e.preventDefault();
handleRetrieve();
}
}}
autoSize={{ minRows: 6, maxRows: 12 }}
style={{
background: colors.bgContainer,
resize: 'none',
}}
/>
<Flex justify="space-between" align="center">
<span style={{
fontSize: 12,
color: colors.textTertiary,
}}>
{searchQuery.length} / 200
</span>
<Button
type="primary"
icon={<SearchOutlined />}
onClick={handleRetrieve}
loading={retrieving}
>
</Button>
</Flex>
</Flex>
{/* 检索设置 */}
<Flex vertical gap={12}>
<span style={{
fontSize: 13,
color: colors.text,
fontWeight: 500,
}}>
</span>
<Flex align="center" gap={12}>
<span style={{
fontSize: 13,
color: colors.textSecondary,
whiteSpace: 'nowrap',
}}>
(Top K):
</span>
<Slider
value={topK}
onChange={setTopK}
min={1}
max={20}
style={{ width: 120 }}
style={{ flex: 1 }}
/>
<span className="option-value">{topK}</span>
</div>
</div>
</Card>
<span style={{
fontSize: 13,
color: colors.text,
minWidth: 24,
textAlign: 'right',
}}>
{topK}
</span>
</Flex>
</Flex>
</Flex>
{/* 检索结果 */}
<div className="retrieve-results">
{/* 右侧面板 - 结果展示 */}
<Flex
vertical
flex={1}
gap={16}
style={{
padding: 20,
background: colors.bgElevated,
overflow: 'auto',
}}
>
{retrieving ? (
<div className="loading-state">
<Flex
flex={1}
align="center"
justify="center"
vertical
gap={12}
>
<Spin size="large" />
<div className="loading-text">...</div>
</div>
<span style={{ color: colors.textSecondary }}>
...
</span>
</Flex>
) : retrieveResults.length === 0 ? (
<Empty
description="请输入关键词进行检索"
className="empty-state"
/>
<Flex
flex={1}
align="center"
justify="center"
vertical
gap={12}
>
<FileSearchOutlined style={{
fontSize: 48,
color: colors.textQuaternary,
}} />
<span style={{ color: colors.textTertiary }}>
</span>
</Flex>
) : (
<>
<div className="results-header">
<span> {retrieveResults.length} </span>
</div>
<Table
columns={columns}
dataSource={retrieveResults}
rowKey={(record) => record.segment.id}
pagination={false}
size="small"
/>
<Flex justify="space-between" align="center">
<span style={{
fontSize: 14,
color: colors.text,
fontWeight: 500,
}}>
</span>
<span style={{
fontSize: 13,
color: colors.textSecondary,
}}>
{retrieveResults.length}
</span>
</Flex>
<Flex vertical gap={12}>
{retrieveResults.map((record, index) => (
<ResultItem
key={record.segment.id}
record={record}
index={index}
/>
))}
</Flex>
</>
)}
</div>
</div>
</Flex>
</Flex>
);
}