From b3d5b3fc8f84aa33d1b3203d896e0f38801894ac Mon Sep 17 00:00:00 2001 From: WangCham <651122857@qq.com> Date: Thu, 30 Oct 2025 12:37:09 +0800 Subject: [PATCH] fix: langchain error --- pkg/rag/knowledge/services/chunker.py | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/pkg/rag/knowledge/services/chunker.py b/pkg/rag/knowledge/services/chunker.py index f169d5f1..19b1f296 100644 --- a/pkg/rag/knowledge/services/chunker.py +++ b/pkg/rag/knowledge/services/chunker.py @@ -4,6 +4,7 @@ import json from typing import List from pkg.rag.knowledge.services import base_service from pkg.core import app +from langchain_text_splitters import RecursiveCharacterTextSplitter class Chunker(base_service.BaseService): @@ -27,21 +28,6 @@ class Chunker(base_service.BaseService): """ if not text: return [] - # words = text.split() - # chunks = [] - # current_chunk = [] - - # for word in words: - # current_chunk.append(word) - # if len(current_chunk) > self.chunk_size: - # chunks.append(" ".join(current_chunk[:self.chunk_size])) - # current_chunk = current_chunk[self.chunk_size - self.chunk_overlap:] - - # if current_chunk: - # chunks.append(" ".join(current_chunk)) - - # A more robust chunking strategy (e.g., using recursive character text splitter) - from langchain.text_splitter import RecursiveCharacterTextSplitter text_splitter = RecursiveCharacterTextSplitter( chunk_size=self.chunk_size,