feat: make CONCURRENCY_POOL_SIZE configurable

.env.example

@@ -61,6 +61,9 @@ MINIO_PORT=9000
 
 MINIO_CONSOLE_PORT=9001
 
+# Embedding并发数，出现429错误时，可调小此参数
+CONCURRENCY_POOL_SIZE=5
+
 # 如果使用ElasticSearch作为向量存储，需要配置以下参数
 # ElasticSearch地址，例如 http://localhost:9200
 # ELASTICSEARCH_ADDR=your_elasticsearch_addr

docker-compose.yml

@@ -44,6 +44,7 @@ services:
       - REDIS_PREFIX=${REDIS_PREFIX}
       - ENABLE_GRAPH_RAG=${ENABLE_GRAPH_RAG}
       - TENANT_AES_KEY=${TENANT_AES_KEY}
+      - CONCURRENCY_POOL_SIZE=${CONCURRENCY_POOL_SIZE:-5}
       - INIT_LLM_MODEL_NAME=${INIT_LLM_MODEL_NAME}
       - INIT_LLM_MODEL_BASE_URL=${INIT_LLM_MODEL_BASE_URL}
       - INIT_LLM_MODEL_API_KEY=${INIT_LLM_MODEL_API_KEY}

docker/Dockerfile.docreader

@@ -91,25 +91,26 @@ RUN apt-get update && apt-get install -y \
     libcups2 \
     libglu1-mesa \
     libsm6 \
+    libreoffice \
     && rm -rf /var/lib/apt/lists/*
 
-# 下载并安装 LibreOffice（区分架构）
+# # 下载并安装 LibreOffice（区分架构）
-RUN mkdir -p /tmp/libreoffice && cd /tmp/libreoffice && \
+# RUN mkdir -p /tmp/libreoffice && cd /tmp/libreoffice && \
-    if [ "$(uname -m)" = "x86_64" ]; then \
+#     if [ "$(uname -m)" = "x86_64" ]; then \
-        wget https://mirrors.tuna.tsinghua.edu.cn/libreoffice/libreoffice/stable/25.2.5/deb/x86_64/LibreOffice_25.2.5_Linux_x86-64_deb.tar.gz && \
+#         wget https://mirrors.tuna.tsinghua.edu.cn/libreoffice/libreoffice/stable/25.2.5/deb/x86_64/LibreOffice_25.2.5_Linux_x86-64_deb.tar.gz && \
-        tar -xzf LibreOffice_25.2.5_Linux_x86-64_deb.tar.gz && \
+#         tar -xzf LibreOffice_25.2.5_Linux_x86-64_deb.tar.gz && \
-        cd LibreOffice_25.2.5.2_Linux_x86-64_deb/DEBS && dpkg -i *.deb; \
+#         cd LibreOffice_25.2.5.2_Linux_x86-64_deb/DEBS && dpkg -i *.deb; \
-    elif [ "$(uname -m)" = "aarch64" ] || [ "$(uname -m)" = "arm64" ]; then \
+#     elif [ "$(uname -m)" = "aarch64" ] || [ "$(uname -m)" = "arm64" ]; then \
-        wget https://mirrors.aliyun.com/libreoffice/testing/25.8.0/deb/aarch64/LibreOffice_25.8.0.3_Linux_aarch64_deb.tar.gz && \
+#         wget https://mirrors.aliyun.com/libreoffice/testing/25.8.0/deb/aarch64/LibreOffice_25.8.0.3_Linux_aarch64_deb.tar.gz && \
-        tar -xzf LibreOffice_25.8.0.3_Linux_aarch64_deb.tar.gz && \
+#         tar -xzf LibreOffice_25.8.0.3_Linux_aarch64_deb.tar.gz && \
-        cd LibreOffice_25.8.0.3_Linux_aarch64_deb/DEBS && dpkg -i *.deb; \
+#         cd LibreOffice_25.8.0.3_Linux_aarch64_deb/DEBS && dpkg -i *.deb; \
-    else \
+#     else \
-        echo "Unsupported architecture: $(uname -m)" && exit 1; \
+#         echo "Unsupported architecture: $(uname -m)" && exit 1; \
-    fi && \
+#     fi && \
-    cd / && rm -rf /tmp/libreoffice
+#     cd / && rm -rf /tmp/libreoffice
 
 # 设置 LibreOffice 环境变量
-RUN echo 'export LIBREOFFICE_PATH=/opt/libreoffice25.2/program/soffice' >> /etc/environment;
+# RUN echo 'export LIBREOFFICE_PATH=/opt/libreoffice25.2/program/soffice' >> /etc/environment;
     
 # 从构建阶段复制已安装的依赖和生成的代码
 COPY --from=builder /usr/local/lib/python3.10/site-packages /usr/local/lib/python3.10/site-packages

internal/container/container.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"os"
 	"slices"
+	"strconv"
 	"strings"
 	"time"
 
@@ -318,10 +319,17 @@ func initRetrieveEngineRegistry(db *gorm.DB, cfg *config.Config) (interfaces.Ret
 //   - Configured goroutine pool
 //   - Error if initialization fails
 func initAntsPool(cfg *config.Config) (*ants.Pool, error) {
-	// Default to 50 if not specified in config
+	// Default to 5 if not specified in config
-	poolSize := 300
+	poolSize := os.Getenv("CONCURRENCY_POOL_SIZE")
+	if poolSize == "" {
+		poolSize = "5"
+	}
+	poolSizeInt, err := strconv.Atoi(poolSize)
+	if err != nil {
+		return nil, err
+	}
 	// Set up the pool with pre-allocation for better performance
-	return ants.NewPool(poolSize, ants.WithPreAlloc(true))
+	return ants.NewPool(poolSizeInt, ants.WithPreAlloc(true))
 }
 
 // registerPoolCleanup registers the goroutine pool for cleanup

services/docreader/src/parser/storage.py

@@ -203,23 +203,20 @@ class MinioStorage(Storage):
         prefer those values to override envs.
         """
         try:
-            if self.storage_config and self.storage_config.get("endpoint") and self.storage_config.get("bucket_name"):
+            endpoint = os.getenv("MINIO_ENDPOINT")
+            use_ssl = os.getenv("MINIO_USE_SSL", "false").lower() == "true"
+            if self.storage_config and self.storage_config.get("bucket_name"):
                 storage_config = self.storage_config
-                # Direct fields
-                endpoint = storage_config.get("endpoint")
                 bucket_name = storage_config.get("bucket_name")
                 path_prefix = storage_config.get("path_prefix").strip().strip("/")
                 access_key = storage_config.get("access_key_id")
                 secret_key = storage_config.get("secret_access_key")
             else:
-                endpoint = os.getenv("MINIO_ENDPOINT")
                 access_key = os.getenv("MINIO_ACCESS_KEY_ID")
                 secret_key = os.getenv("MINIO_SECRET_ACCESS_KEY")
                 bucket_name = os.getenv("MINIO_BUCKET_NAME")
-                use_ssl = os.getenv("MINIO_USE_SSL", "false").lower() == "true"
                 path_prefix = os.getenv("MINIO_PATH_PREFIX", "").strip().strip("/")
 
-
             if not all([endpoint, access_key, secret_key, bucket_name]):
                 logger.error("Incomplete MinIO configuration, missing required environment variables")
                 return None, None, None, None, None