self::$uid]) ->when($params['name'], function ($query) use($params) { $query->where('name', 'like', '%'. $params['name']. '%'); }) ->select() ->each(function ($item) { $item->file_count = KnowledgeFile::where(['index_id' => $item['index_id']])->count(); }) ->toArray(); $data = [ 'lists' => $result, 'count' => Knowledge::where(['user_id' => self::$uid]) ->when($params['name'], function ($query) use($params) { $query->where('name', 'like', '%'. $params['name']. '%'); })->count(), 'page_no' => $params['page_no'] ?? 1, 'page_size' => $params['page_size'] ?? 10, ]; return $data; } /** * 知识库创建 * * @param array $data * @return void */ public static function add(array $params){ if(!isset($params['name']) || empty($params['name'])){ throw new \Exception('知识库名称不能为空'); } if(mb_strlen($params['name'], "UTF-8") > 12){ throw new \Exception('知识名称不能超过12个字符'); } if(mb_strlen($params['description'], "UTF-8") > 1000){ throw new \Exception('知识库描述不能超过1000个字符'); } $find = Knowledge::where('name', $params['name'])->where('user_id', self::$uid)->limit(1)->find(); if (!empty($find)){ message('知识库名称已存在'); } if(strpos($params['description'], $_SERVER['HTTP_HOST']) === false){ $params['description'] = $params['description']; } $params['site'] = $_SERVER['HTTP_HOST']; # 检查创建知识库的算力是否足够,不够则提示 //计费 $unit = TokenLogService::checkToken(self::$uid, self::KNOELEDGE_CREATE); # 1 创建分类 $knowledgeNmae = $params['name'].bin2hex(random_bytes(4)); $cateRes = \app\common\service\ToolsService::Knowledge()->createCategory([ 'name' => $knowledgeNmae ]); if((int)$cateRes['code'] === 10000){ if(isset($cateRes['data']['status']) && (int)$cateRes['data']['status'] === 1){ message($cateRes['data']['msg']); return false; } # 2 创建同名分类知识库 $params['category_id'] = $cateRes['data']['CategoryId']; $params['task_id'] = generate_unique_task_id(); $request = $params; $request['name'] = $knowledgeNmae; $request['description'] = $params['site'] . ' '.$params['description']; $indexRes = self::requestUrl($request, self::KNOELEDGE_CREATE, self::$uid); if($indexRes){ unset($params['task_id']); $addData = $params; $addData['user_id'] = self::$uid; $addData['index_id'] = $indexRes['Id']; $addData['category_id'] = $cateRes['data']['CategoryId']; $addData['create_time'] = time(); $addData['status'] = 1; $addData['is_bind'] = 0; $addData['rerank_min_score'] = $params['rerank_min_score'] ?? self::RERANK_MIN_SCORE; $detial = new Knowledge(); if($detial->save($addData)){ if(!empty($params['documents'])){ $file_ids = array_column($params['documents'], 'file_id'); $addDocRes = \app\common\service\ToolsService::Knowledge()->addDocJobIndex([ 'indexid' => $indexRes['Id'], 'documentIds' => $file_ids, ]); if((int)$addDocRes['code'] === 10000){ $docs = array_map(function($doc) use ($detial, $params, $indexRes){ $_tmp = array(); $_tmp['file_id'] = $doc['file_id']; $_tmp['category_id'] = $doc['category_id'] ?? ''; $_tmp['status'] = $doc['status'] ?? 'PARSE_SUCCESS'; $_tmp['type'] = $doc['type'] ?? ''; $_tmp['name'] = $doc['name']; $_tmp['size'] = $doc['size'] ?? 0; $_tmp['parser'] = $doc['parser'] ?? 'DASHSCOPE_DOCMIND'; $_tmp['user_id'] = self::$uid; $_tmp['index_id'] = $indexRes['Id']; $_tmp['kid'] = $detial->id; $_tmp['file_url'] = $doc['uri'] ?? ($doc['file_url'] ?? ''); $_tmp['create_time'] = time(); return $_tmp; }, $params['documents']); if(!empty($docs)){ $detialFile = new KnowledgeFile(); $detialFile->saveAll($docs); } }else{ message($addDocRes['message']); } } self::$returnData = $detial->toArray(); return true; } }else{ message('知识库创建失败'); } }else{ message($cateRes['message']); } } public static function edit(array $params){ $find = Knowledge::where('id', $params['id'])->where('user_id', self::$uid)->findOrEmpty(); if ($find->isEmpty()){ message('知识库不存在'); } if(mb_strlen($params['name'], "UTF-8") > 12){ throw new \Exception('知识名称不能超过12个字符'); } if(mb_strlen($params['description'], "UTF-8") > 1000){ message('知识库描述不能超过1000个字符'); } $row = Knowledge::where('name', $params['name'])->where('id', '<>', $params['id'])->where('user_id', self::$uid)->limit(1)->find(); if (!empty($row)){ message('知识库名称重复'); } // if($params['overlap_size'] > $params['chunk_size']){ // message('分段预估长度必须大于分段重叠长度'); // } $addData = $params; $addData['update_time'] = time(); unset($addData['create_time']); if($find->save($addData)){ } self::$returnData = $find->toArray(); return true; } public static function edit1(array $params){ $find = Knowledge::where('id', $params['id'])->where('user_id', self::$uid)->findOrEmpty(); if ($find->isEmpty()){ message('知识库不存在'); } if(strlen($params['description']) > 1000){ message('知识库描述不能超过1000个字符'); } $addData = $params; $addData['update_time'] = time(); unset($addData['create_time']); if($find->save($addData)){ KnowledgeFile::where('user_id', '=', self::$uid)->where('kid', '=', $params['id'])->select()->delete(); if(!empty($params['documents'])){ $file_ids = array_column($params['documents'], 'file_id'); //先删除在添加 $delDocRes = \app\common\service\ToolsService::Knowledge()->deleteDocIndex([ 'indexid' => $find['index_id'], 'documentids' => $file_ids, ]); if((int)$delDocRes['code'] === 10000){ $addDocRes = \app\common\service\ToolsService::Knowledge()->addDocJobIndex([ 'indexid' => $find['index_id'], 'documentIds' => $file_ids, ]); if((int)$addDocRes['code'] !== 10000){ message($addDocRes['message']); } $docs = array_map(function($doc) use ($find, $params){ $_tmp['file_id'] = $doc['file_id']; $_tmp['category_id'] = $doc['category_id'] ?? ''; $_tmp['status'] = $doc['status'] ?? 'PARSE_SUCCESS'; $_tmp['type'] = $doc['type'] ?? ''; $_tmp['name'] = $doc['name']; $_tmp['size'] = $doc['size'] ?? 0; $_tmp['parser'] = $doc['parser'] ?? 'DASHSCOPE_DOCMIND'; $_tmp['user_id'] = self::$uid; $_tmp['index_id'] = $find['index_id']; $_tmp['kid'] = $find->id; $_tmp['file_url'] = $doc['uri'] ?? ($doc['file_url'] ?? ''); $_tmp['create_time'] = time(); return $_tmp; }, $params['documents']); if(!empty($docs)){ $detialFile = new KnowledgeFile(); $detialFile->saveAll($docs); } }else{ message($delDocRes['message']); } } } self::$returnData = $find->toArray(); return true; } public static function detail(array $param){ $find = Knowledge::where('id', $param['id'])->limit(1)->find(); if(empty($find)){ message('知识库不存在'); } $files = KnowledgeFile::field('*')->where(['user_id' => self::$uid])->where('kid', $find['id'])->select() ->toArray(); if(!empty($files)){ $find->is_bind = 1; $find->update_time = time(); $find->save(); } $result = $find->toArray(); $result['documents'] = $files; return $result; } /** * 知识库删除 * * @param array $params * @return void */ public static function delete(array $params){ $find = Knowledge::where('id', $params['id'])->where('user_id', self::$uid)->fetchSql(false)->limit(1)->find(); if (empty($find)){ message('知识库不存在'); } try { // 请求查询接口 $response = \app\common\service\ToolsService::Knowledge()->deleteIndex([ 'id' => $find['index_id'], ]); $cateRes = \app\common\service\ToolsService::Knowledge()->deleteCategory([ 'categoryid' => $find['category_id'], ]); if((int)$response['code'] !== 10000){ message($response['message']); } Knowledge::destroy(['id' => $params['id'], 'user_id' => self::$uid]); KnowledgeFile::destroy(['kid' => $params['id'], 'user_id' => self::$uid]); return true; } catch (\Throwable $e) { return false; } } /** * 知识库检索 * * @param array $params * @return void */ public static function retrieve(array $params){ if(!isset($params['prompt']) || empty($params['prompt'])){ message('提示词 不能为空'); } try { // 请求查询接口 $response = \app\common\service\ToolsService::Knowledge()->retrieveIndex($params); if((int)$response['code'] !== 10000){ message($response['message']); } return self::__insertRetrieveData($params, $response['data']); } catch (\Throwable $e) { message($e->getMessage()); } } private static function __insertRetrieveData(array $params, array $data){ $knowledge = Knowledge::where('index_id', $params['indexid'])->where('user_id', self::$uid)->fetchSql(false)->limit(1)->find(); if(empty($knowledge)){ message('知识库不存在'); } try { $retrieveData = [ 'user_id' => self::$uid, 'kid' => $knowledge['id'], 'index_id' => $params['indexid'], 'rerank_min_score' => $params['rerank_min_score'] ?? self::RERANK_MIN_SCORE, // 默认为0 'prompt' => $params['prompt'], 'create_time' => time(), ]; $_retrieve = new KnowledgeRetrieve(); if($_retrieve->save($retrieveData)){ if(isset($data['Nodes']) && !empty($data['Nodes'])){ $items = array_map(function($item) use ($_retrieve, $params){ $hash = hash_hmac('sha256', $item['Text'], 'knowledge'); $_row = KnowledgeRetrieveSlice::field('content, hash') ->where('index_id', $params['indexid']) ->where('user_id', self::$uid) ->where('rid', $_retrieve->id) ->where('hash', $hash) ->limit(1)->find(); if(empty($_row)){ $filename = KnowledgeFile::field('name') ->where('file_id', $item['Metadata']['doc_id']) ->where('index_id', $params['indexid']) //->where('user_id', self::$uid) ->limit(1)->value('name'); $tmp = [ 'user_id' => self::$uid, 'rid' => $_retrieve->id, 'index_id' => $params['indexid'], 'content' => $item['Text'], 'score' => $item['Score'], 'hash' => $hash, 'metadata' => json_encode($item['Metadata'], JSON_UNESCAPED_UNICODE), // 假设Metadata是一个数组,你可以根据实际情况调整 'create_time' => time(), 'source' => $filename ?? $item['Metadata']['doc_name'], ]; return $tmp; } }, $data['Nodes']); $items = array_filter($items); if(!empty($items)){ $_slice = new KnowledgeRetrieveSlice(); $_slice->saveAll($items); } return $items; } } return []; } catch (\Exception $e) { message($e->getMessage()); } } public static function historyTest(array $params){ $params['page_no'] = $params['page_no'] ?? 1; $params['page_size'] = $params['page_size'] ?? 15; $pageSize = $params['page_size']; $pageNo = ($params['page_no'] - 1) * $params['page_size']; $keywords = $params['keywords']?? ''; if(!isset($params['indexid']) || empty($params['indexid'])){ message('知识库ID 不能为空'); } $list = KnowledgeRetrieve::where('user_id', self::$uid) ->where('index_id', $params['indexid']) ->when($keywords, function ($query) use ($keywords) { $query->where('prompt', 'like', '%' . $keywords . '%'); }) ->limit($pageNo, $pageSize) ->order('create_time', 'desc') ->select() ->toArray(); $data = [ 'lists' => $list, 'count' => KnowledgeRetrieve::where('user_id', self::$uid) ->where('index_id', $params['indexid']) ->when($keywords, function ($query) use ($keywords) { $query->where('prompt', 'like', '%' . $keywords . '%'); })->count(), 'page_no' => $params['page_no'], 'page_size' => $params['page_size'], ]; return $data; } public static function testDetail(array $params){ $params['page_no'] = $params['page_no'] ?? 1; $params['page_size'] = $params['page_size'] ?? 15; $pageSize = $params['page_size']; $pageNo = ($params['page_no'] - 1) * $params['page_size']; if(!isset($params['id']) || empty($params['id'])){ message('检索ID 不能为空'); } $list = KnowledgeRetrieveSlice::where('user_id', self::$uid) ->where('rid', $params['id']) ->order('create_time', 'desc') ->limit($pageNo, $pageSize) ->select() ->toArray(); $data = [ 'lists' => $list, 'count' => KnowledgeRetrieveSlice::where('user_id', self::$uid) ->where('rid', $params['id'])->count(), 'page_no' => $params['page_no'], 'page_size' => $params['page_size'], ]; return $data; } /** * 知识库详情-文档列表 * * @param array $params * @return void */ public static function indexFileList(array $param){ $find = Knowledge::where('id', $param['id'])->limit(1)->find(); if(empty($find)){ message('知识库不存在'); } $result = KnowledgeFile::where(['user_id' => self::$uid])->where('index_id', $find['index_id'])->select() ->toArray(); $data = [ 'lists' => $result, 'count' => KnowledgeFile::where(['user_id' => self::$uid])->where('index_id', $find['index_id'])->count(), 'page_no' => $params['page_no'] ?? 1, 'page_size' => $params['page_size'] ?? 10, ]; return $data; } /** * 知识库分片 * * @param array $params * @return void */ public static function chunkLists(array $params){ try { // 请求查询接口 $response = \app\common\service\ToolsService::Knowledge()->chunkIndex($params); if((int)$response['code'] !== 10000){ message($response['message']); } return $response['data']; } catch (\Throwable $e) { message($e->getMessage()); } } public static function fileUpload(array $params){ if(!isset($params['indexid']) || empty($params['indexid'])){ message('请选择知识库'); } $index = Knowledge::where('index_id', $params['indexid'])->limit(1)->find(); if(empty($index)){ message('知识库不存在'); } $file = request()->file('file'); $exts = [ 'doc', 'docx', 'wps', 'ppt', 'pptx', 'xls', 'xlsx', 'md', 'txt', 'pdf', 'png', 'jpg', 'jpeg', 'bmp', 'gif','aac', 'amr', 'flac', 'flv', 'm4a', 'mp3', 'mpeg', 'ogg', 'opus', 'wav', 'webm', 'wma','mp4', 'mkv', 'avi', 'mov', 'wmv' ]; if($file){ $ext = $file->getOriginalExtension(); if(!in_array($ext, $exts)){ message('知识库暂时不支持csv文件'); } } try { $result = UploadService::file(0, self::$uid, FileEnum::SOURCE_USER, 'uploads/file/knowledge'); if(!empty($result)){ $response = \app\common\service\ToolsService::Knowledge()->createFile([ 'file_url' => $result['uri'], 'category_id' => $index['category_id'] ]); if((int)$response['code'] === 10000){ $fileinfo = \app\common\service\ToolsService::Knowledge()->infoFile([ 'id' => $response['data']['FileId'] ]); if((int)$fileinfo['code'] === 10000){ $result['category_id'] = $fileinfo['data']['CategoryId']; $result['status'] = $fileinfo['data']['Status']; $result['type'] = $fileinfo['data']['FileType']; $result['name'] = $result['name'] ?? $fileinfo['data']['FileName']; $result['size'] = $fileinfo['data']['SizeInBytes']; $result['parser'] = $fileinfo['data']['Parser']; } $result['file_id'] = $response['data']['FileId']; $result['parser'] = $response['data']['Parser']; return $result; }else{ message($response['message']); } }else{ message('文件上传失败'); } } catch (\Exception $e) { message($e->getMessage()); } } public static function fileLists(array $params){ $pageNo = ($params['page_no'] - 1) * $params['page_size']; $pageSize = $params['page_size']; $name = $params['name']?? ''; $takeover_mode = $params['takeover_mode']?? ''; $modes = array( 0 => 'PARSING', 1 => 'PARSE_SUCCESS', 2 => 'PARSE_FAILED' ); $status = $modes[$takeover_mode] ?? ''; // 默认为0,即未完成的任务,你可以根据需要修改这个值 $result = KnowledgeFile::where(['user_id' => self::$uid]) ->where('category_id', $params['category_id']) ->when($name, function ($query) use ($name) { $query->where('name', 'like', '%' . $name . '%'); }) ->when($status, function ($query) use ($status) { $query->where('status', '=', $status); }) ->limit($pageNo, $pageSize) ->select() ->toArray(); $data = [ 'lists' => $result, 'count' => KnowledgeFile::where(['user_id' => self::$uid]) ->when($name, function ($query) use ($name) { $query->where('name', 'like', '%' . $name . '%'); }) ->when($status, function ($query) use ($status) { $query->where('status', '=', $status); }) ->where('category_id', $params['category_id'])->count(), 'page_no' => $params['page_no'], 'page_size' => $params['page_size'], ]; return $data; } public static function fileAdd(array $params){ $find = Knowledge::where('category_id', $params['category_id'])->limit(1)->find(); if(empty($find)){ message('知识库不存在'); } $addData = array( 'description' => $params['description'] ?? $find['description'], 'rerank_min_score' => $params['rerank_min_score'] ?? ($find['rerank_min_score'] ?? self::RERANK_MIN_SCORE), // 默认为0 'separator' => $params['separator'] ?? $find['separator'], 'chunk_size' => $params['chunk_size'] ?? $find['chunk_size'], 'overlap_size' => $params['overlap_size'] ?? $find['overlap_size'], 'structure_type' => $params['structure_type'] ?? $find['structure_type'], 'source_type' => $params['source_type'] ?? $find['source_type'], 'sink_type' => $params['sink_type'] ?? $find['sink_type'], 'strategy' => $params['strategy'] ?? $find['strategy'], 'is_bind' => !empty($params['documents']) ? 1: 0, 'site' => $params['site'] ?? $_SERVER['HTTP_HOST'], 'update_time' => time(), ); // if($addData['overlap_size'] > $addData['chunk_size']){ // message('分段预估长度必须大于分段重叠长度'); // } if(!$find->save($addData)){ message('知识库信息更新失败'); } if(!empty($params['documents'])){ foreach($params['documents'] as $key => $val){ $file = KnowledgeFile::where('file_id', $val['file_id'])->where('user_id', self::$uid)->limit(1)->find(); if(!empty($file)){ unset($params['documents'][$key]); } } if(empty($params['documents'])){ message('请上传文件'); } $fileids = array_column($params['documents'], 'file_id'); $addDocRes = \app\common\service\ToolsService::Knowledge()->addDocJobIndex([ 'indexid' => $find['index_id'], 'documentIds' => $fileids ]); if((int)$addDocRes['code'] === 10000){ if(isset($addDocRes['data']['status']) && (int)$addDocRes['data']['status'] === 400){ $find->delete(); message('知识库数据异常:' . $addDocRes['data']['msg']); } $docs = array_map(function($doc) use ($find, $params){ $_tmp = array(); $_tmp['file_id'] = $doc['file_id']; $_tmp['category_id'] = $params['category_id']; $_tmp['status'] = $doc['status'] ?? 'INIT'; $_tmp['type'] = $doc['type'] ?? ''; $_tmp['name'] = $doc['name']; $_tmp['size'] = $doc['size'] ?? 0; $_tmp['parser'] = $doc['parser'] ?? 'DASHSCOPE_DOCMIND'; $_tmp['user_id'] = self::$uid; $_tmp['index_id'] = $find['index_id']; $_tmp['kid'] = $find->id; $_tmp['file_url'] = $doc['uri'] ?? ($doc['file_url'] ?? ''); $_tmp['create_time'] = time(); return $_tmp; }, $params['documents']); if(!empty($docs)){ $detialFile = new KnowledgeFile(); $detialFile->saveAll($docs); } return true; }else{ message($addDocRes['message']); } } return true; } public static function fileDetial(array $params){ $find = KnowledgeFile::where(['user_id' => self::$uid])->where('file_id', $params['file_id'])->limit(1)->find(); if(empty($find)){ message('文件不存在'); } if($find['status'] !== 'PARSE_SUCCESS'){ $fileinfo = \app\common\service\ToolsService::Knowledge()->infoFile([ 'id' => $find['file_id'] ]); if((int)$fileinfo['code'] === 10000){ $find['status'] = $fileinfo['data']['Status']; $find['update_time'] = time(); $find['type'] = $fileinfo['data']['FileType']; $find['name'] = $fileinfo['data']['FileName']; $find['size'] = $fileinfo['data']['SizeInBytes']; $find['parser'] = $fileinfo['data']['Parser']; $find->save(); } } return $find->toArray(); } public static function fileDelete(array $params){ $find = KnowledgeFile::where('id', $params['id'])->where('user_id', self::$uid)->fetchSql(false)->limit(1)->find(); if (empty($find)){ message('文档不存在'); } KnowledgeFile::destroy(['id' => $params['id'], 'user_id' => self::$uid]); // 请求查询接口 $response = \app\common\service\ToolsService::Knowledge()->deleteFile([ 'id' => $find['file_id'] ]); // if((int)$response['code'] !== 10000){ // //throw new \Exception($response['message']); // message($response['message']); // } return true; } /** * 知识库分片 * * @param array $params * @return void */ public static function fileChunkLists(array $params){ $find = KnowledgeFile::where('id', $params['id'])->where('user_id', self::$uid)->fetchSql(false)->limit(1)->find(); if (empty($find)){ message('文档不存在'); } $params['page_no'] = $params['page_no'] ?? 1; $params['page_size'] = $params['page_size'] ?? 15; $pageSize = $params['page_size']; $pageNo = ($params['page_no'] - 1) * $params['page_size']; $keywords = $params['keywords']?? ''; $result = KnowledgeFileSlice::where('file_id', $find['file_id']) ->where('index_id', $find['index_id']) ->where('user_id', self::$uid) ->when($keywords, function ($query) use ($keywords) { $query->where('content', 'like', '%' . $keywords . '%'); }) ->limit($pageNo, $pageSize) ->select()->toArray(); if(empty($result)){ // 请求查询接口 $response = \app\common\service\ToolsService::Knowledge()->chunkIndex([ 'indexid' => $find['index_id'], 'fileid' => $find['file_id'], 'pageNum' => 1, 'pageSize' => 100, ]); if((int)$response['code'] === 10000){ if(!isset($response['data']['Nodes'])){ message('查询失败'); } $nodes = $response['data']['Nodes']; if(!empty($nodes)){ $items = array_map(function($item) use ($find, $params){ $hash = hash_hmac('sha256', $item['Text'], 'knowledge'); $_row = KnowledgeFileSlice::field('content, hash') ->where('file_id', $find['file_id']) ->where('index_id', $find['index_id']) ->where('user_id', self::$uid) ->where('hash', $hash) ->limit(1)->find(); if(empty($_row)){ $tmp = [ 'user_id' => self::$uid, 'rid' => $find->id, 'index_id' => $find['index_id'], 'file_id' => $find['file_id'], 'content' => $item['Text'], 'score' => $item['Score'], 'hash' => $hash, 'metadata' => json_encode($item['Metadata'], JSON_UNESCAPED_UNICODE), // 假设Metadata是一个数组,你可以根据实际情况调整 'create_time' => time(), 'source' => $find['name'] ?? $item['Metadata']['doc_name'] ]; return $tmp; } }, $nodes); $items = array_filter($items); if(!empty($items)){ $_slice = new KnowledgeFileSlice(); $_slice->saveAll($items); } } } $result = KnowledgeFileSlice::where('user_id', self::$uid) ->where('index_id', $find['index_id']) ->where('file_id', $find['file_id']) ->when($keywords, function ($query) use ($keywords){ $query->where('content', 'like', '%' . $keywords . '%'); }) ->limit($pageNo, $pageSize) ->select() ->each(function ($item) { $item['metadata'] = json_decode($item['metadata'], true); // 假设metadata是一个JSON字符串 return $item; }) ->toArray(); } $data = [ 'lists' => $result, 'count' => KnowledgeFileSlice::where('user_id', self::$uid) ->where('index_id', $find['index_id']) ->where('file_id', $find['file_id']) ->when($keywords, function ($query) use ($keywords){ $query->where('content', 'like', '%' . $keywords . '%'); })->count(), 'page_no' => $params['page_no'], 'page_size' => $params['page_size'], ]; return $data; } public static function updateTagFile(array $params){ try { // 请求查询接口 $response = \app\common\service\ToolsService::Knowledge()->updateTagFile($params); if((int)$response['code'] !== 10000){ throw new \Exception($response['message']); } return $response['data']; } catch (\Throwable $e) { throw new \Exception($e->getMessage()); } } /** * 绑定知识库 * * @param array $params * @param [type] $data * @return void */ public static function bind(array $params, $data){ //删除原来的绑定 添加新的绑定信息 KnowledgeBind::where('data_id', $data['id']) ->where('user_id', self::$uid) ->where('type', $params['type']) ->select() ->delete(); if(isset($params['index_id']) && !empty($params['index_id'])){ $knowledge = Knowledge::where('index_id', $params['index_id'])->limit(1)->find(); if(empty($knowledge)){ throw new \Exception('知识库不存在'); return false; } //挂载知识库 KnowledgeBind::create([ 'user_id' => self::$uid, 'kid' => $knowledge['id'], 'data_id' => $data['id'], 'type' => $params['type'], 'index_id' => $params['index_id'], 'rerank_min_score' => $params['rerank_min_score'] ?? self::RERANK_MIN_SCORE, 'create_time' => time(), ]); } } /** * 将训练内容上传到知识库 * * @param array $params * @return void */ public static function ladderPlayerUpload(array $params){ # 1 根据已经选择的数据生成文本文件 # 2 上传文件到分类 # 3 文档追加到指定的知识库 # 4 获取文件的信息 if(!isset($params['ids']) || empty($params['ids'])){ message('请选择需要上传的记录'); } $knowledge = Knowledge::where('index_id', $params['indexid'])->where('user_id', self::$uid)->limit(1)->find(); if(empty($knowledge)){ message('知识库不存在'); } # 提取文本信息 $data = \app\common\model\lianlian\LlChat::alias('c') ->field('c.ask, c.reply, c.performance, c.speechcraft') ->join('ll_analysis l', 'l.id = c.analysis_id', 'left') ->where('l.id', 'in', $params['ids']) ->select()->toArray(); try { $content = array(); foreach($data as $item){ $content[] = $item['ask'] . "\n" . $item['reply'] . "\n" . $item['performance'] . "\n" . $item['speechcraft']; } $content = implode("\n", $content); $uri = 'uploads/file/knowledge/'.date('Ymd').'/话术1.txt'; $filepath = root_path(). '/public/' . $uri; !is_dir(dirname($filepath)) && mkdir(dirname($filepath), 0777, true); $size = file_put_contents($filepath, $content); if($size === false){ message('文件写入失败'); } $uri = config('app.app_host') . '/' . $uri; $fileRes = \app\common\service\ToolsService::Knowledge()->createFile([ 'file_url' => $uri, 'category_id' => $knowledge['category_id'] ]); if((int)$fileRes['code'] === 10000){ $fileinfo = \app\common\service\ToolsService::Knowledge()->infoFile([ 'id' => $fileRes['data']['FileId'] ]); if((int)$fileinfo['code'] === 10000){ $addDocRes = \app\common\service\ToolsService::Knowledge()->addDocJobIndex([ 'indexid' => $knowledge['index_id'], 'documentIds' => [$fileRes['data']['FileId']] ]); if((int)$addDocRes['code'] === 10000){ $result = array(); $result['file_id'] = $fileRes['data']['FileId']; $result['category_id'] = $knowledge['category_id']; $result['status'] = $fileinfo['data']['Status']; $result['type'] = $fileinfo['data']['FileType']; $result['name'] = $result['name'] ?? $fileinfo['data']['FileName']; $result['size'] = $fileinfo['data']['SizeInBytes']; $result['parser'] = $fileinfo['data']['Parser']; $result['user_id'] = self::$uid; $result['index_id'] = $knowledge['index_id']; $result['kid'] = $knowledge->id; $result['file_url'] = $uri; $result['create_time'] = time(); $detialFile = new KnowledgeFile(); $detialFile->save($result); self::$returnData = $result; return true; }else{ message('文件追加到知识库失败'); } }else{ message('获取文件信息失败'); } }else{ message('上传文件失败'); } } catch (\Throwable $th) { message($th->getMessage()); } return false; } /******************请求接口以及扣费************************ */ /** * 请求上游接口与计费 * @param array $request * @param string $scene * @param int $userId * @param string $taskId * @return array * @throws \Exception */ private static function requestUrl(array $request, string $scene, int $userId, array $record = []): array { $requestService = \app\common\service\ToolsService::Knowledge(); [$tokenScene, $tokenCode] = match ($scene) { self::KNOELEDGE_CREATE => ['knowledge_create', AccountLogEnum::TOKENS_DEC_KNOWLEDGE_CREATE], self::KNOELEDGE_RETRIEVE => ['knowledge_retrieve', AccountLogEnum::TOKENS_DEC_KNOWLEDGE_RETRIEVE], self::KNOELEDGE_CHAT => ['knowledge_chat', AccountLogEnum::TOKENS_DEC_KNOWLEDGE_CHAT], }; //计费 $unit = TokenLogService::checkToken($userId, $tokenScene); switch ($scene) { case self::KNOELEDGE_CREATE: $response = $requestService->createIndex($request); break; case self::KNOELEDGE_RETRIEVE: $response = $requestService->retrievePrompt($request); break; case self::KNOELEDGE_CHAT: $response = $requestService->promptChat($request); break; default: } //print_r($response);die; if($scene == self::KNOELEDGE_CHAT && $request['stream'] == true){ exit; } //clogger('response: '. json_encode($response, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES)); //成功响应,需要扣费 if (isset($response['code']) && $response['code'] == 10000) { if($scene === self::KNOELEDGE_CREATE){ $tokens = $unit; $points = $unit; $knowlwdge_tokens = $points; }else{ $usage = $response['data']['usage']; $tokens = $usage['total_tokens'] + $request['knowledge_tokens']; //计算消耗tokens $points = $unit > 0 ? ceil($tokens / $unit) : 0; $knowlwdge_tokens = $request['knowledge_tokens']; } //clogger($points.'|'.$userId); if ($points > 0) { $extra = ['总消耗tokens数' => $tokens, '知识库消耗tokens数' => $knowlwdge_tokens, '算力单价' => $unit, '实际消耗算力' => $points]; //clogger('extra: '. json_encode($extra, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES)); //token扣除 User::userTokensChange($userId, $points); //记录日志 AccountLogLogic::recordUserTokensLog(true, $userId, $tokenCode, $points, $request['task_id'], $extra); self::saveKnowledgeRecord($request, $response['data']); } } return $response['data'] ?? []; } public static function saveKnowledgeRecord(array $request, array $response){ try { $record = $request['knowledge_record']; $record['content'] = $response['content'] ?? ( $response['choices'][0]['message']['content']?? ''); $record['prompt_tokens'] = $response['usage']['prompt_tokens']; $record['completion_tokens'] = $response['usage']['completion_tokens']; $record['total_tokens'] = $response['usage']['total_tokens']; $record['tokens'] = $record['retrieve_tokens'] + $record['total_tokens']; $record['task_id'] = $request['task_id']; $record['create_time'] = time(); $res = (new KnowledgeUseSceneRecord())->save($record); $knowlwdge = Knowledge::where('index_id', $request['indexid'])->where('user_id', $request['user_id'])->fetchSql(false)->limit(1)->find(); //请求成功知识库调用+1 $knowlwdge->inc('request_count', 1)->inc('tokens', $record['retrieve_tokens'])->save(); } catch (\Throwable $th) { //clogger($th); } } /******************定时任务-文件状态************************ */ public static function setFileStatus(){ try { // clogger('setFileStatus ' . date('Y-m-d H:i:s', time()), 'cron'); $files = KnowledgeFile::where('status', 'not in',['PARSE_SUCCESS', 'PARSE_FAILED'])->order('id asc')->limit(10)->select(); foreach ($files as $key => $file) { //clogger('开始处理文件:' . $file['file_id'], 'cron'); $fileinfo = \app\common\service\ToolsService::Knowledge()->infoFile([ 'id' => $file['file_id'] ]); if((int)$fileinfo['code'] === 10000){ if(isset($fileinfo['data']['no_exist']) && (int)$fileinfo['data']['no_exist'] === 1){ $file['status'] = 'PARSE_FAILED'; $file['remark'] = $fileinfo['data']['msg']; //clogger('文件不存在:'. $file['file_id'], 'cron'); }else{ $file['status'] = $fileinfo['data']['Status']; } $file['update_time'] = time(); $file->save(); } sleep(2); } return true; } catch (\Throwable $th) { //clogger('文件处理失败:'. $th->getMessage(), 'cron'); return false; } } public static function fileChunksPull(){ try { //clogger('fileChunksPull:' . date('Y-m-d H:i:s', time()), 'cron'); $files = KnowledgeFile::where('status', 'PARSE_SUCCESS') ->where('is_completed', 0) ->order('update_time asc')->limit(5)->select(); foreach ($files as $key => $file) { //clogger('开始处理文件:'. $file['file_id'], 'cron'); # 首次拉取 # 按照分页拉取 $totalRes = \app\common\service\ToolsService::Knowledge()->chunkIndex([ 'indexid' => $file['index_id'], 'fileid' => $file['file_id'], 'pageNum' => 1, 'pageSize' => 1, ]); if((int)$totalRes['code'] === 10000){ if(isset($totalRes['data']['no_exist']) && (int)$totalRes['data']['no_exist'] === 1){ $file['status'] = 'PARSE_FAILED'; //clogger('文件或知识库不存在:'. $file['file_id'].'||'. $file['index_id'], 'cron'); //删除本地文件 $file['status'] = 'PARSE_FAILED'; $file['remark'] = $totalRes['data']['msg']; $file['is_completed'] = 1; $file['update_time'] = time(); $file['delete_time'] = time(); $file->save(); }else{ $total = $totalRes['data']['Total']; $totalPage = ceil($total / 100); for($page = 1; $page <= $totalPage; $page++){ $response = \app\common\service\ToolsService::Knowledge()->chunkIndex([ 'indexid' => $file['index_id'], 'fileid' => $file['file_id'], 'pageNum' => $page, 'pageSize' => 100, ]); if((int)$response['code'] === 10000){ $nodes = $response['data']['Nodes']; if(!empty($nodes)){ $items = array_map(function($item) use ($file){ $hash = hash_hmac('sha256', $item['Text'], 'knowledge'); $_row = KnowledgeFileSlice::field('content, hash') ->where('file_id', $file['file_id']) ->where('index_id', $file['index_id']) ->where('user_id', $file['user_id']) ->where('hash', $hash) ->limit(1)->find(); if(empty($_row)){ $tmp = [ 'user_id' => $file['user_id'], 'rid' => $file->id, 'index_id' => $file['index_id'], 'file_id' => $file['file_id'], 'content' => $item['Text'], 'score' => $item['Score'], 'hash' => $hash, 'metadata' => json_encode($item['Metadata'], JSON_UNESCAPED_UNICODE), // 假设Metadata是一个数组,你可以根据实际情况调整 'create_time' => time(), 'source' => $file['name'] ?? $item['Metadata']['doc_name'] ]; return $tmp; } }, $nodes); $items = array_filter($items); if(!empty($items)){ $_slice = new KnowledgeFileSlice(); $_slice->saveAll($items); } } } sleep(2); } $file['is_completed'] = 1; $file['slice_count'] = $total; $file['update_time'] = time(); $file->save(); } } sleep(2); } return true; } catch (\Throwable $th) { //clogger('文件处理失败:'. $th->getMessage(), 'cron'); return false; } } /******************chat************************ */ public static function chat($params){ set_time_limit(0); if(!isset($params['message']) || empty($params['message'])){ message('提示词 不能为空'); } $uid = $params['user_id'] ?? self::$uid; $knowlwdge = Knowledge::where('index_id', $params['indexid'])->where('user_id', $uid)->fetchSql(false)->limit(1)->find(); if(empty($knowlwdge)){ message('知识库不存在'); } $message = $params['message']; // 表单变量替换 $message_ext = $params['message_ext'] ?? ''; if ($message_ext) { $message_ext_text = self::parseMsg($message_ext, ''); $message = $message_ext_text . $message; } $request = [ 'indexid' => $params['indexid'], 'prompt' => $message, 'rerank_min_score' => $params['rerank_min_score'] ?? ($knowledge['rerank_min_score'] ?? self::RERANK_MIN_SCORE), // 默认为0 'stream' => (bool)$params['stream'] ?? true, 'task_id' => $params['task_id'] ?? generate_unique_task_id(), 'scene' => $params['scene'] ?? '未知聊天', 'assistant_id' => $params['assistant_id'] ?? 0 ]; if($params['scene'] == '陪练聊天'){ $request['voice'] = $params['voice']?? ''; $request['emotion'] = $params['emotion']?? ''; $request['intensity'] = $params['intensity']?? ''; } self::__getRequestData($request, $params); try { $record = array( 'user_id' => $uid, 'index_id' => $params['indexid'], 'prompt' => $message, 'rerank_min_score' => $params['rerank_min_score'] ?? self::RERANK_MIN_SCORE, // 默认为0 'scene' => $params['scene'] ?? '未知聊天', ); // 根据用户提示词检索 $response = \app\common\service\ToolsService::Knowledge()->retrievePrompt($request); // 拼接切片内容 if((int)$response['code'] === 10000){ if(isset($response['data']['Nodes'])){ $texts = implode("\n", array_column($response['data']['Nodes'], 'Text')); }else{ $texts = ''; } $textLength = mb_strlen($texts, 'utf-8'); $record['retrieve_content'] = $texts; $record['retrieve_length'] = $textLength; $record['retrieve_tokens'] = ceil($textLength / 4); //2个字一个token $prompt = "请根据以下知识库内容回答问题: {$texts} 问题:{$message}"; $request['user_id'] = $uid; // 替换为实际的用户ID $request['prompt'] = $prompt; $request['knowledge_tokens'] = ceil($textLength / 4); $request['chat_type'] = 9006; $request['now'] = time(); $request['knowledge_record'] = $record; $result = self::requestUrl($request, self::KNOELEDGE_CHAT, $uid, $record); self::$returnData = $result; return $result; } return true; } catch (\Throwable $e) { // clogger($e); if($params['scene'] !== 'socket'){ message($e->getMessage()); }else{ self::$returnData = []; return false; } } } public static function socketChat($params){ set_time_limit(0); try { if(!isset($params['message']) || empty($params['message'])){ return [false, '提示词 不能为空']; } $uid = $params['user_id'] ?? self::$uid; $knowlwdge = Knowledge::where('index_id', $params['indexid'])->where('user_id', $uid)->fetchSql(false)->limit(1)->find(); if(empty($knowlwdge)){ return [false, '知识库不存在']; } $message = $params['message']; // 表单变量替换 $message_ext = $params['message_ext'] ?? ''; if ($message_ext) { $message_ext_text = self::parseMsg($message_ext, ''); $message = $message_ext_text . $message; } $request = [ 'indexid' => $params['indexid'], 'prompt' => $message, 'rerank_min_score' => $params['rerank_min_score'] ?? ($knowledge['rerank_min_score'] ?? self::RERANK_MIN_SCORE), // 默认为0 'stream' => (bool)$params['stream'] ?? true, 'task_id' => $params['task_id'] ?? generate_unique_task_id(), 'scene' => $params['scene'] ?? '未知聊天', 'assistant_id' => $params['assistant_id'] ?? 0 ]; if($params['scene'] == '陪练聊天'){ $request['voice'] = $params['voice']?? ''; $request['emotion'] = $params['emotion']?? ''; $request['intensity'] = $params['intensity']?? ''; } self::__getRequestData($request, $params); $record = array( 'user_id' => $uid, 'index_id' => $params['indexid'], 'prompt' => $message, 'rerank_min_score' => $params['rerank_min_score'] ?? self::RERANK_MIN_SCORE, // 默认为0 'scene' => $params['scene'] ?? '未知聊天', ); // 根据用户提示词检索 $response = \app\common\service\ToolsService::Knowledge()->retrievePrompt($request); // 拼接切片内容 if((int)$response['code'] === 10000){ if(isset($response['data']['Nodes'])){ $texts = implode("\n", array_column($response['data']['Nodes'], 'Text')); }else{ $texts = ''; } $textLength = mb_strlen($texts, 'utf-8'); $record['retrieve_content'] = $texts; $record['retrieve_length'] = $textLength; $record['retrieve_tokens'] = ceil($textLength / 4); //2个字一个token $prompt = "请根据以下知识库内容回答问题: {$texts} 问题:{$message}"; $request['user_id'] = $uid; // 替换为实际的用户ID $request['prompt'] = $prompt; $request['knowledge_tokens'] = ceil($textLength / 4); $request['chat_type'] = 9006; $request['now'] = time(); $request['knowledge_record'] = $record; $result = self::requestUrl($request, self::KNOELEDGE_CHAT, $uid, $record); return [true, $result]; }else{ return [false, $response['message'] ?? '知识库检索失败']; } } catch (\Throwable $e) { return [false, $e->getMessage()]; } } public static function sceneChat($params){ set_time_limit(0); if (empty($params['message']) && empty($params['message_ext'])) { message('提示词 不能为空'); } $assistant = \app\common\model\chat\Assistants::where('id', $params['assistant_id'])->findOrEmpty(); if ($assistant->isEmpty()) { message('助手不存在'); } $uid = $params['user_id'] ?? self::$uid; $knowlwdge = Knowledge::where('index_id', $params['indexid'])->where('user_id', $uid)->fetchSql(false)->limit(1)->find(); if(empty($knowlwdge)){ message('知识库不存在'); } $message = $params['message']; $request = [ 'indexid' => $params['indexid'], 'prompt' => $message, 'rerank_min_score' => $params['rerank_min_score'] ?? ($knowlwdge['rerank_min_score'] ?? self::RERANK_MIN_SCORE), // 默认为0 'stream' => (bool)$params['stream'] ?? true, 'task_id' => $params['task_id'] ?? generate_unique_task_id(), 'scene' => $params['scene'] ?? '未知聊天', 'assistant_id' => $params['assistant_id'] ?? 0 ]; //clogger(json_encode($request, JSON_UNESCAPED_UNICODE | JSON_PRETTY_PRINT)); if(mb_strlen(mb_trim($message), 'utf-8') == 0){ message('提示词 不能为空'); } if($params['scene'] == '陪练聊天'){ $request['voice'] = $params['voice']?? ''; $request['emotion'] = $params['emotion']?? ''; $request['intensity'] = $params['intensity']?? ''; } self::__getRequestData($request, $params); try { $record = array( 'user_id' => $uid, 'index_id' => $params['indexid'], 'prompt' => $message, 'rerank_min_score' => $request['rerank_min_score'], // 默认为0 'scene' => $params['scene'] ?? '未知聊天', ); // 根据用户提示词检索 $response = \app\common\service\ToolsService::Knowledge()->retrievePrompt($request); // 拼接切片内容 if((int)$response['code'] === 10000){ if(isset($response['data']['Nodes'])){ $texts = implode("\n", array_column($response['data']['Nodes'], 'Text')); }else{ $texts = ''; } //clogger($texts); // 表单变量替换 $message_ext = $params['message_ext'] ?? ''; if ($message_ext) { $message_ext_text = self::parseMsg($message_ext, $assistant['form_info']);// $assistant['form_info'] $texts = $message_ext_text . $texts; } $textLength = mb_strlen($texts, 'utf-8'); $record['retrieve_content'] = $texts; $record['retrieve_length'] = $textLength; $record['retrieve_tokens'] = ceil($textLength / 4); //2个字一个token $prompt = "请根据以下知识库内容回答问题: {$texts} 问题:{$message}"; $request['user_id'] = $uid; // 替换为实际的用户ID $request['prompt'] = $prompt; $request['knowledge_tokens'] = ceil($textLength / 4); $request['chat_type'] = 9006; $request['now'] = time(); $request['knowledge_record'] = $record; //clogger(json_encode($request, JSON_UNESCAPED_UNICODE | JSON_PRETTY_PRINT)); $result = self::requestUrl($request, self::KNOELEDGE_CHAT, $uid, $record); self::$returnData = $result; return $result; } return true; } catch (\Throwable $e) { message($e->getMessage()); } } public static function ladderPlayerChat($params){ set_time_limit(0); if (empty($params['message']) && empty($params['message_ext'])) { message('提示词 不能为空'); } $uid = $params['user_id'] ?? self::$uid; $knowlwdge = Knowledge::where('index_id', $params['indexid'])->where('user_id', $uid)->fetchSql(false)->limit(1)->find(); if(empty($knowlwdge)){ message('知识库不存在'); } $message = $params['message']; $request = [ 'indexid' => $params['indexid'], 'prompt' => $message, 'rerank_min_score' => $params['rerank_min_score'] ?? ($knowlwdge['rerank_min_score'] ?? self::RERANK_MIN_SCORE), // 默认为0 'stream' => (bool)$params['stream'] ?? true, 'task_id' => $params['task_id'] ?? generate_unique_task_id(), 'scene' => $params['scene'] ?? '未知聊天', 'assistant_id' => $params['assistant_id'] ?? 0 ]; $request['rerank_min_score'] = self::RERANK_MIN_SCORE; $request['voice'] = $params['voice']?? ''; $request['emotion'] = $params['emotion']?? ''; $request['intensity'] = $params['intensity']?? ''; self::__getRequestData($request, $params); try { $record = array( 'user_id' => $uid, 'index_id' => $params['indexid'], 'prompt' => $message, 'rerank_min_score' => $request['rerank_min_score'], // 默认为0 'scene' => $params['scene'] ?? '未知聊天', ); //clogger(json_encode($request, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES), 'll'); // 根据用户提示词检索 $response = \app\common\service\ToolsService::Knowledge()->retrievePrompt($request); //clogger(json_encode($response, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES), 'll'); // 拼接切片内容 if((int)$response['code'] === 10000){ if(isset($response['data']['Nodes'])){ $texts = implode("\n", array_column($response['data']['Nodes'], 'Text')); }else{ $texts = ''; } $textLength = mb_strlen($texts, 'utf-8'); $record['retrieve_content'] = $texts; $record['retrieve_length'] = $textLength; $record['retrieve_tokens'] = ceil($textLength / 4); //2个字一个token $prompt = "请根据以下知识库内容回答问题: {$texts} 问题:{$message}"; $request['user_id'] = $uid; // 替换为实际的用户ID $request['prompt'] = $prompt; $request['knowledge_tokens'] = ceil($textLength / 4); $request['chat_type'] = 9006; $request['now'] = time(); $request['knowledge_record'] = $record; $result = self::requestUrl($request, self::KNOELEDGE_CHAT, $uid, $record); self::$returnData = $result; return $result; } return true; } catch (\Throwable $e) { message($e->getMessage()); } } private static function parseMsg($message_ext, $form_info){ $message_ext = json_decode($message_ext, true); if (empty($message_ext)) { return ''; } preg_match_all('/\${([^\}]+)}/u', $form_info, $matches); $keys = $matches[1]; if (empty($keys)) { return ''; } foreach ($message_ext as $key => $value) { foreach ($keys as $keyword) { if ($keyword == $key) { if (!empty($value) && is_array($value)) { $value = implode(',', $value); } $form_info = str_replace('${' . $keyword . '}', $value, $form_info); } } } return $form_info; } private static function __getRequestData(array &$request, array $params){ if(isset($params['rerankTopN']) && !empty($params['rerankTopN'])){ $request['rerankTopN'] = $params['rerankTopN']; } if(isset($params['denseSimilarityTopK']) && !empty($params['denseSimilarityTopK'])){ $request['denseSimilarityTopK'] = $params['denseSimilarityTopK']; } if(isset($params['enableReranking']) && !empty($params['enableReranking'])){ $request['enableReranking'] = $params['enableReranking']; } if(isset($params['enableRewrite']) && !empty($params['enableRewrite'])){ $request['enableRewrite'] = $params['enableRewrite']; } if(isset($params['sparseSimilarityTopK']) && !empty($params['sparseSimilarityTopK'])){ $request['sparseSimilarityTopK'] = $params['sparseSimilarityTopK']; } if(isset($params['saveRetrieverHistory']) && !empty($params['saveRetrieverHistory'])){ $request['saveRetrieverHistory'] = $params['saveRetrieverHistory']; } } }