diff --git a/libs/dify_service_api/v1/client.py b/libs/dify_service_api/v1/client.py index 35defe2c..244d701e 100644 --- a/libs/dify_service_api/v1/client.py +++ b/libs/dify_service_api/v1/client.py @@ -5,6 +5,8 @@ import typing import json from .errors import DifyAPIError +from pathlib import Path +import os class AsyncDifyServiceClient: @@ -109,7 +111,23 @@ class AsyncDifyServiceClient: user: str, timeout: float = 30.0, ) -> str: - """上传文件""" + # 处理 Path 对象 + if isinstance(file, Path): + if not file.exists(): + raise ValueError(f'File not found: {file}') + with open(file, 'rb') as f: + file = f.read() + + # 处理文件路径字符串 + elif isinstance(file, str): + if not os.path.isfile(file): + raise ValueError(f'File not found: {file}') + with open(file, 'rb') as f: + file = f.read() + + # 处理文件对象 + elif hasattr(file, 'read'): + file = file.read() async with httpx.AsyncClient( base_url=self.base_url, trust_env=True, @@ -121,6 +139,8 @@ class AsyncDifyServiceClient: headers={'Authorization': f'Bearer {self.api_key}'}, files={ 'file': file, + }, + data={ 'user': (None, user), }, ) diff --git a/libs/dingtalk_api/api.py b/libs/dingtalk_api/api.py index 1731d38a..03caabfd 100644 --- a/libs/dingtalk_api/api.py +++ b/libs/dingtalk_api/api.py @@ -188,12 +188,80 @@ class DingTalkClient: if incoming_message.message_type == 'richText': data = incoming_message.rich_text_content.to_dict() + + # 使用统一的结构化数据格式，保持顺序 + rich_content = { + 'Type': 'richText', + 'Elements': [], # 按顺序存储所有元素 + 'SimpleContent': '', # 兼容字段：纯文本内容 + 'SimplePicture': '' # 兼容字段：第一张图片 + } + + # 先收集所有文本和图片占位符 + text_elements = [] + image_placeholders = [] + + # 解析富文本内容，保持原始顺序 for item in data['richText']: - if 'text' in item: - message_data['Content'] = item['text'] - if incoming_message.get_image_list()[0]: - message_data['Picture'] = await self.download_image(incoming_message.get_image_list()[0]) - message_data['Type'] = 'text' + + # 处理文本内容 + if 'text' in item and item['text'] != "\n": + element = { + 'Type': 'text', + 'Content': item['text'] + } + rich_content['Elements'].append(element) + text_elements.append(item['text']) + + # 检查是否是图片元素 - 根据钉钉API的实际结构调整 + # 钉钉富文本中的图片通常有特定标识，可能需要根据实际返回调整 + elif item.get("type") == "picture": + # 创建图片占位符 + element = { + 'Type': 'image_placeholder', + } + rich_content['Elements'].append(element) + + # 获取并下载所有图片 + image_list = incoming_message.get_image_list() + if image_list: + new_elements = [] + image_index = 0 + + for element in rich_content['Elements']: + if element['Type'] == 'image_placeholder': + if image_index < len(image_list) and image_list[image_index]: + image_url = await self.download_image(image_list[image_index]) + new_elements.append({ + 'Type': 'image', + 'Picture': image_url + }) + image_index += 1 + else: + # 如果没有对应的图片，保留占位符或跳过 + continue + else: + new_elements.append(element) + + rich_content['Elements'] = new_elements + + + # 设置兼容字段 + all_texts = [elem['Content'] for elem in rich_content['Elements'] if elem.get('Type') == 'text'] + rich_content['SimpleContent'] = '\n'.join(all_texts) if all_texts else '' + + all_images = [elem['Picture'] for elem in rich_content['Elements'] if elem.get('Type') == 'image'] + if all_images: + rich_content['SimplePicture'] = all_images[0] + rich_content['AllImages'] = all_images # 所有图片的列表 + + # 设置原始的 content 和 picture 字段以保持兼容 + message_data['Content'] = rich_content['SimpleContent'] + message_data['Rich_Content'] = rich_content + if all_images: + message_data['Picture'] = all_images[0] + + elif incoming_message.message_type == 'text': message_data['Content'] = incoming_message.get_text_list()[0] diff --git a/libs/dingtalk_api/dingtalkevent.py b/libs/dingtalk_api/dingtalkevent.py index adc4ea7c..f45264c6 100644 --- a/libs/dingtalk_api/dingtalkevent.py +++ b/libs/dingtalk_api/dingtalkevent.py @@ -15,6 +15,10 @@ class DingTalkEvent(dict): def content(self): return self.get('Content', '') + @property + def rich_content(self): + return self.get('Rich_Content', '') + @property def incoming_message(self) -> Optional['dingtalk_stream.chatbot.ChatbotMessage']: return self.get('IncomingMessage') diff --git a/pkg/persistence/migrations/dbm011_dify_base_prompt_config.py b/pkg/persistence/migrations/dbm011_dify_base_prompt_config.py new file mode 100644 index 00000000..a98c8050 --- /dev/null +++ b/pkg/persistence/migrations/dbm011_dify_base_prompt_config.py @@ -0,0 +1,40 @@ +from .. import migration + +import sqlalchemy + +from ...entity.persistence import pipeline as persistence_pipeline + + +@migration.migration_class(11) +class DBMigrateDifyApiConfig(migration.DBMigration): + """Langflow API config""" + + async def upgrade(self): + """Upgrade""" + # read all pipelines + pipelines = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(persistence_pipeline.LegacyPipeline)) + + for pipeline in pipelines: + serialized_pipeline = self.ap.persistence_mgr.serialize_model(persistence_pipeline.LegacyPipeline, pipeline) + + config = serialized_pipeline['config'] + + if 'base-prompt' not in config['ai']['dify-service-api']: + config['ai']['dify-service-api']['base-prompt'] = ( + 'When the file content is readable, please read the content of this file. When the file is an image, describe the content of this image.', + ) + + await self.ap.persistence_mgr.execute_async( + sqlalchemy.update(persistence_pipeline.LegacyPipeline) + .where(persistence_pipeline.LegacyPipeline.uuid == serialized_pipeline['uuid']) + .values( + { + 'config': config, + 'for_version': self.ap.ver_mgr.get_current_version(), + } + ) + ) + + async def downgrade(self): + """Downgrade""" + pass diff --git a/pkg/platform/sources/dingtalk.py b/pkg/platform/sources/dingtalk.py index d5d85997..ed468e35 100644 --- a/pkg/platform/sources/dingtalk.py +++ b/pkg/platform/sources/dingtalk.py @@ -1,3 +1,4 @@ + import traceback import typing from libs.dingtalk_api.dingtalkevent import DingTalkEvent @@ -36,16 +37,31 @@ class DingTalkMessageConverter(abstract_platform_adapter.AbstractMessageConverte if atUser.dingtalk_id == event.incoming_message.chatbot_user_id: yiri_msg_list.append(platform_message.At(target=bot_name)) - if event.content: - text_content = event.content.replace('@' + bot_name, '') - yiri_msg_list.append(platform_message.Plain(text=text_content)) - if event.picture: - yiri_msg_list.append(platform_message.Image(base64=event.picture)) + if event.rich_content: + elements = event.rich_content.get("Elements") + for element in elements: + if element.get('Type') == 'text': + text = element.get('Content', '').replace('@' + bot_name, '') + if text.strip(): + yiri_msg_list.append(platform_message.Plain(text=text)) + elif element.get('Type') == 'image' and element.get('Picture'): + yiri_msg_list.append(platform_message.Image(base64=element['Picture'])) + else: + # 回退到原有简单逻辑 + if event.content: + text_content = event.content.replace('@' + bot_name, '') + yiri_msg_list.append(platform_message.Plain(text=text_content)) + if event.picture: + yiri_msg_list.append(platform_message.Image(base64=event.picture)) + + # 处理其他类型消息（文件、音频等） if event.file: yiri_msg_list.append(platform_message.File(url=event.file, name=event.name)) if event.audio: yiri_msg_list.append(platform_message.Voice(base64=event.audio)) + + chain = platform_message.MessageChain(yiri_msg_list) return chain diff --git a/pkg/provider/runners/difysvapi.py b/pkg/provider/runners/difysvapi.py index b98a9bc3..eefc7524 100644 --- a/pkg/provider/runners/difysvapi.py +++ b/pkg/provider/runners/difysvapi.py @@ -14,6 +14,7 @@ import langbot_plugin.api.entities.builtin.pipeline.query as pipeline_query from libs.dify_service_api.v1 import client, errors + @runner.runner_class('dify-service-api') class DifyServiceAPIRunner(runner.RequestRunner): """Dify Service API 对话请求器""" @@ -77,7 +78,7 @@ class DifyServiceAPIRunner(runner.RequestRunner): tuple[str, list[str]]: 纯文本和图片的 Dify 服务图片 ID """ plain_text = '' - image_ids = [] + file_ids = [] if isinstance(query.user_message.content, list): for ce in query.user_message.content: @@ -92,11 +93,24 @@ class DifyServiceAPIRunner(runner.RequestRunner): f'{query.session.launcher_type.value}_{query.session.launcher_id}', ) image_id = file_upload_resp['id'] - image_ids.append(image_id) + file_ids.append(image_id) + # elif ce.type == "file_url": + # file_bytes = base64.b64decode(ce.file_url) + # file_upload_resp = await self.dify_client.upload_file( + # file_bytes, + # f'{query.session.launcher_type.value}_{query.session.launcher_id}', + # ) + # file_id = file_upload_resp['id'] + # file_ids.append(file_id) elif isinstance(query.user_message.content, str): plain_text = query.user_message.content + # plain_text = "When the file content is readable, please read the content of this file. When the file is an image, describe the content of this image." if file_ids and not plain_text else plain_text + # plain_text = "The user message type cannot be parsed." if not file_ids and not plain_text else plain_text + # plain_text = plain_text if plain_text else "When the file content is readable, please read the content of this file. When the file is an image, describe the content of this image." + # print(self.pipeline_config['ai']) + plain_text = plain_text if plain_text else self.pipeline_config['ai']['dify-service-api']['base-prompt'] - return plain_text, image_ids + return plain_text, file_ids async def _chat_messages( self, query: pipeline_query.Query @@ -110,7 +124,6 @@ class DifyServiceAPIRunner(runner.RequestRunner): files = [ { 'type': 'image', - 'transfer_method': 'local_file', 'upload_file_id': image_id, } for image_id in image_ids diff --git a/pkg/utils/constants.py b/pkg/utils/constants.py index 5a7ab4fa..56fa2e4d 100644 --- a/pkg/utils/constants.py +++ b/pkg/utils/constants.py @@ -1,6 +1,6 @@ semantic_version = 'v4.4.1' -required_database_version = 10 +required_database_version = 11 """Tag the version of the database schema, used to check if the database needs to be migrated""" debug_mode = False diff --git a/templates/metadata/pipeline/ai.yaml b/templates/metadata/pipeline/ai.yaml index f6d54ee6..7a13b2b1 100644 --- a/templates/metadata/pipeline/ai.yaml +++ b/templates/metadata/pipeline/ai.yaml @@ -124,6 +124,16 @@ stages: zh_Hans: 基础 URL type: string required: true + - name: base-prompt + label: + en_US: Base PROMPT + zh_Hans: 基础提示词 + description: + en_US: When Dify receives a message with empty input (only images), it will pass this default prompt into it. + zh_Hans: 当 Dify 接收到输入文字为空（仅图片）的消息时，传入该默认提示词 + type: string + required: true + default: "When the file content is readable, please read the content of this file. When the file is an image, describe the content of this image." - name: app-type label: en_US: App Type