From d7b354b9b48c3166e349e3b01c082f02a3d32992 Mon Sep 17 00:00:00 2001 From: Bruce Date: Sat, 6 Sep 2025 14:01:42 +0000 Subject: [PATCH 1/4] add database connect config --- pkg/persistence/databases/postgresql.py | 30 +++++++++++++++++++++++++ pkg/persistence/databases/sqlite.py | 4 ++-- pkg/persistence/mgr.py | 7 ++++-- templates/config.yaml | 5 ++++- 4 files changed, 41 insertions(+), 5 deletions(-) create mode 100644 pkg/persistence/databases/postgresql.py diff --git a/pkg/persistence/databases/postgresql.py b/pkg/persistence/databases/postgresql.py new file mode 100644 index 00000000..19b3706b --- /dev/null +++ b/pkg/persistence/databases/postgresql.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +import pip +import sqlalchemy.ext.asyncio as sqlalchemy_asyncio +import sys + +from .. import database + + +@database.manager_class('postgresql') +class PostgreSQLDatabaseManager(database.BaseDatabaseManager): + """PostgreSQL database manager""" + + async def initialize(self) -> None: + + # default to PostgreSQL with asyncpg driver + try: + __import__("asyncpg") + except ImportError: + print('以下依赖包未安装，将自动安装，请完成后重启程序：') + print( + 'The dependence package asyncpg is missing, it will be installed automatically, please restart the program after completion:' + ) + pip.main(['install', "asyncpg"]) + print('已自动安装缺失的依赖包 asyncpg ，请重启程序。') + print('The missing dependence asyncpg have been installed automatically, please restart the program.') + sys.exit(0) + + engine_url = self.ap.instance_config.data['system'].get('database', {}).get('engine_url', 'postgresql+asyncpg://root:***@127.0.0.1:5432/postgres') + self.engine = sqlalchemy_asyncio.create_async_engine(engine_url) diff --git a/pkg/persistence/databases/sqlite.py b/pkg/persistence/databases/sqlite.py index c1337459..7b219e4c 100644 --- a/pkg/persistence/databases/sqlite.py +++ b/pkg/persistence/databases/sqlite.py @@ -10,5 +10,5 @@ class SQLiteDatabaseManager(database.BaseDatabaseManager): """SQLite database manager""" async def initialize(self) -> None: - sqlite_path = 'data/langbot.db' - self.engine = sqlalchemy_asyncio.create_async_engine(f'sqlite+aiosqlite:///{sqlite_path}') + engine_url = self.ap.instance_config.data['system'].get('database', {}).get('engine_url', 'sqlite+aiosqlite:///data/langbot.db') + self.engine = sqlalchemy_asyncio.create_async_engine(engine_url) diff --git a/pkg/persistence/mgr.py b/pkg/persistence/mgr.py index 9b926733..06ccc859 100644 --- a/pkg/persistence/mgr.py +++ b/pkg/persistence/mgr.py @@ -38,9 +38,12 @@ class PersistenceManager: async def initialize(self): self.ap.logger.info('Initializing database...') + database_type = self.ap.instance_config.data['system'].get('database', {}).get('type', 'sqlite') for manager in database.preregistered_managers: - self.db = manager(self.ap) - await self.db.initialize() + if manager.name == database_type: + self.db = manager(self.ap) + await self.db.initialize() + break await self.create_tables() diff --git a/templates/config.yaml b/templates/config.yaml index 8c0ac685..dbb5cc46 100644 --- a/templates/config.yaml +++ b/templates/config.yaml @@ -20,6 +20,9 @@ system: jwt: expire: 604800 secret: '' +database: + type: sqlite + engine_url: 'sqlite+aiosqlite:///data/langbot.db' vdb: use: chroma qdrant: @@ -30,4 +33,4 @@ vdb: plugin: runtime_ws_url: 'ws://langbot_plugin_runtime:5400/control/ws' enable_marketplace: true - cloud_service_url: 'https://space.langbot.app' + cloud_service_url: 'https://space.langbot.app' \ No newline at end of file From 0b0a0c07a0d83882e2b860917c811408867d84f5 Mon Sep 17 00:00:00 2001 From: Bruce Date: Sun, 7 Sep 2025 10:52:41 +0000 Subject: [PATCH 2/4] add adapter send card & dingtalk send_card --- libs/dingtalk_api/api.py | 53 ++++++++++++++++++++++++++++++++ pkg/platform/sources/dingtalk.py | 6 ++++ 2 files changed, 59 insertions(+) diff --git a/libs/dingtalk_api/api.py b/libs/dingtalk_api/api.py index 3d483a3a..daeaefcc 100644 --- a/libs/dingtalk_api/api.py +++ b/libs/dingtalk_api/api.py @@ -253,6 +253,59 @@ class DingTalkClient: await self.logger.error(f'failed to send proactive massage to group: {traceback.format_exc()}') raise Exception(f'failed to send proactive massage to group: {traceback.format_exc()}') + async def send_card(self, target_type: str, target_id: str, card_template_id: str, card_data: dict, + at_sender: bool = False, + at_all: bool = False) -> None: + + # 构造 incoming_message + if target_type == 'group': + conversation_type = "2" + sender_staff_id = "" + conversation_id = target_id + else: + conversation_type = "1" + sender_staff_id = target_id + conversation_id = target_id + + create_at = int(time.time() * 1000) # 毫秒时间戳 + # 计算 sessionWebhookExpiredTime，假设是 createAt 之后的 1 小时 + session_webhook_expired_time = create_at + 3600 * 1000 # 3600 秒 = 1 小时，转换为毫秒 + + incoming_message = dingtalk_stream.ChatbotMessage.from_dict( + { + "conversationId": conversation_id, + "openThreadId": conversation_id, + "senderNick": sender_staff_id, + "isAdmin": True, + "senderStaffId": sender_staff_id, + "sessionWebhookExpiredTime": session_webhook_expired_time, + "createAt": create_at, + "conversationType": str(conversation_type), + "senderId": "", + "robotCode": self.credential.client_id, + } + ) + + card_replier = dingtalk_stream.CardReplier(self.client, incoming_message) + try: + # 发送卡片 + card_instance_id = await card_replier.async_create_and_send_card( + card_template_id=card_template_id, + card_data=card_data, + callback_type="STREAM", + callback_route_key="", + at_sender=at_sender, + at_all=at_all, + recipients=[target_id] if target_type == 'person' else None, + support_forward=True, + ) + if card_instance_id: + await self.logger.info(f'Card sent successfully, card_instance_id: {card_instance_id}') + return + except Exception: + await self.logger.error(f'failed to send card: {traceback.format_exc()}') + raise Exception(f'failed to send card: {traceback.format_exc()}') + async def create_and_card( self, temp_card_id: str, incoming_message: dingtalk_stream.ChatbotMessage, quote_origin: bool = False ): diff --git a/pkg/platform/sources/dingtalk.py b/pkg/platform/sources/dingtalk.py index eb15775f..0ff28650 100644 --- a/pkg/platform/sources/dingtalk.py +++ b/pkg/platform/sources/dingtalk.py @@ -184,6 +184,12 @@ class DingTalkAdapter(abstract_platform_adapter.AbstractMessagePlatformAdapter): if target_type == 'group': await self.bot.send_proactive_message_to_group(target_id, content) + async def send_card(self, target_type: str, target_id: str, card_template_id: str, card_data: dict, + at_sender: bool = False, + at_all: bool = False): + self.ap.logger.info(f'card_data: {card_data}') + await self.bot.send_card(target_type, target_id, card_template_id, card_data, at_sender, at_all) + async def is_stream_output_supported(self) -> bool: is_stream = False if self.config.get('enable-stream-reply', None): From e2071d9486b311a3b996a922ed6d658f8b7a5e4a Mon Sep 17 00:00:00 2001 From: Bruce Date: Sat, 13 Sep 2025 03:00:19 +0000 Subject: [PATCH 3/4] Revert "add adapter send card & dingtalk send_card" This reverts commit 0f4d65072b5e328e7d3c50291a8d5157beff1225. --- libs/dingtalk_api/api.py | 53 -------------------------------- pkg/platform/sources/dingtalk.py | 6 ---- 2 files changed, 59 deletions(-) diff --git a/libs/dingtalk_api/api.py b/libs/dingtalk_api/api.py index daeaefcc..3d483a3a 100644 --- a/libs/dingtalk_api/api.py +++ b/libs/dingtalk_api/api.py @@ -253,59 +253,6 @@ class DingTalkClient: await self.logger.error(f'failed to send proactive massage to group: {traceback.format_exc()}') raise Exception(f'failed to send proactive massage to group: {traceback.format_exc()}') - async def send_card(self, target_type: str, target_id: str, card_template_id: str, card_data: dict, - at_sender: bool = False, - at_all: bool = False) -> None: - - # 构造 incoming_message - if target_type == 'group': - conversation_type = "2" - sender_staff_id = "" - conversation_id = target_id - else: - conversation_type = "1" - sender_staff_id = target_id - conversation_id = target_id - - create_at = int(time.time() * 1000) # 毫秒时间戳 - # 计算 sessionWebhookExpiredTime，假设是 createAt 之后的 1 小时 - session_webhook_expired_time = create_at + 3600 * 1000 # 3600 秒 = 1 小时，转换为毫秒 - - incoming_message = dingtalk_stream.ChatbotMessage.from_dict( - { - "conversationId": conversation_id, - "openThreadId": conversation_id, - "senderNick": sender_staff_id, - "isAdmin": True, - "senderStaffId": sender_staff_id, - "sessionWebhookExpiredTime": session_webhook_expired_time, - "createAt": create_at, - "conversationType": str(conversation_type), - "senderId": "", - "robotCode": self.credential.client_id, - } - ) - - card_replier = dingtalk_stream.CardReplier(self.client, incoming_message) - try: - # 发送卡片 - card_instance_id = await card_replier.async_create_and_send_card( - card_template_id=card_template_id, - card_data=card_data, - callback_type="STREAM", - callback_route_key="", - at_sender=at_sender, - at_all=at_all, - recipients=[target_id] if target_type == 'person' else None, - support_forward=True, - ) - if card_instance_id: - await self.logger.info(f'Card sent successfully, card_instance_id: {card_instance_id}') - return - except Exception: - await self.logger.error(f'failed to send card: {traceback.format_exc()}') - raise Exception(f'failed to send card: {traceback.format_exc()}') - async def create_and_card( self, temp_card_id: str, incoming_message: dingtalk_stream.ChatbotMessage, quote_origin: bool = False ): diff --git a/pkg/platform/sources/dingtalk.py b/pkg/platform/sources/dingtalk.py index 0ff28650..eb15775f 100644 --- a/pkg/platform/sources/dingtalk.py +++ b/pkg/platform/sources/dingtalk.py @@ -184,12 +184,6 @@ class DingTalkAdapter(abstract_platform_adapter.AbstractMessagePlatformAdapter): if target_type == 'group': await self.bot.send_proactive_message_to_group(target_id, content) - async def send_card(self, target_type: str, target_id: str, card_template_id: str, card_data: dict, - at_sender: bool = False, - at_all: bool = False): - self.ap.logger.info(f'card_data: {card_data}') - await self.bot.send_card(target_type, target_id, card_template_id, card_data, at_sender, at_all) - async def is_stream_output_supported(self) -> bool: is_stream = False if self.config.get('enable-stream-reply', None): From 6a7e88ffd6de983b815d47d0f7f8c1a4fb1ac467 Mon Sep 17 00:00:00 2001 From: Junyan Qin Date: Sat, 13 Sep 2025 17:59:10 +0800 Subject: [PATCH 4/4] perf: minor fixes --- pkg/core/bootutils/deps.py | 1 + pkg/persistence/databases/postgresql.py | 23 ++++++------------- pkg/persistence/databases/sqlite.py | 3 ++- pkg/persistence/mgr.py | 5 ++-- .../dbm006_plugin_install_source.py | 18 ++++++++++++--- pyproject.toml | 3 ++- templates/config.yaml | 11 +++++++-- 7 files changed, 38 insertions(+), 26 deletions(-) diff --git a/pkg/core/bootutils/deps.py b/pkg/core/bootutils/deps.py index 1a439af8..b2508b22 100644 --- a/pkg/core/bootutils/deps.py +++ b/pkg/core/bootutils/deps.py @@ -40,6 +40,7 @@ required_deps = { 'sqlmodel': 'sqlmodel', 'telegramify_markdown': 'telegramify-markdown', 'slack_sdk': 'slack_sdk', + 'asyncpg': 'asyncpg', } diff --git a/pkg/persistence/databases/postgresql.py b/pkg/persistence/databases/postgresql.py index 19b3706b..f63d8f61 100644 --- a/pkg/persistence/databases/postgresql.py +++ b/pkg/persistence/databases/postgresql.py @@ -1,8 +1,6 @@ from __future__ import annotations -import pip import sqlalchemy.ext.asyncio as sqlalchemy_asyncio -import sys from .. import database @@ -12,19 +10,12 @@ class PostgreSQLDatabaseManager(database.BaseDatabaseManager): """PostgreSQL database manager""" async def initialize(self) -> None: + postgresql_config = self.ap.instance_config.data.get('database', {}).get('postgresql', {}) - # default to PostgreSQL with asyncpg driver - try: - __import__("asyncpg") - except ImportError: - print('以下依赖包未安装，将自动安装，请完成后重启程序：') - print( - 'The dependence package asyncpg is missing, it will be installed automatically, please restart the program after completion:' - ) - pip.main(['install', "asyncpg"]) - print('已自动安装缺失的依赖包 asyncpg ，请重启程序。') - print('The missing dependence asyncpg have been installed automatically, please restart the program.') - sys.exit(0) - - engine_url = self.ap.instance_config.data['system'].get('database', {}).get('engine_url', 'postgresql+asyncpg://root:***@127.0.0.1:5432/postgres') + host = postgresql_config.get('host', '127.0.0.1') + port = postgresql_config.get('port', 5432) + user = postgresql_config.get('user', 'postgres') + password = postgresql_config.get('password', 'postgres') + database = postgresql_config.get('database', 'postgres') + engine_url = f'postgresql+asyncpg://{user}:{password}@{host}:{port}/{database}' self.engine = sqlalchemy_asyncio.create_async_engine(engine_url) diff --git a/pkg/persistence/databases/sqlite.py b/pkg/persistence/databases/sqlite.py index 7b219e4c..ed096040 100644 --- a/pkg/persistence/databases/sqlite.py +++ b/pkg/persistence/databases/sqlite.py @@ -10,5 +10,6 @@ class SQLiteDatabaseManager(database.BaseDatabaseManager): """SQLite database manager""" async def initialize(self) -> None: - engine_url = self.ap.instance_config.data['system'].get('database', {}).get('engine_url', 'sqlite+aiosqlite:///data/langbot.db') + db_file_path = self.ap.instance_config.data.get('database', {}).get('sqlite', {}).get('path', 'data/langbot.db') + engine_url = f'sqlite+aiosqlite:///{db_file_path}' self.engine = sqlalchemy_asyncio.create_async_engine(engine_url) diff --git a/pkg/persistence/mgr.py b/pkg/persistence/mgr.py index 06ccc859..bab2cde5 100644 --- a/pkg/persistence/mgr.py +++ b/pkg/persistence/mgr.py @@ -36,9 +36,8 @@ class PersistenceManager: self.meta = base.Base.metadata async def initialize(self): - self.ap.logger.info('Initializing database...') - - database_type = self.ap.instance_config.data['system'].get('database', {}).get('type', 'sqlite') + database_type = self.ap.instance_config.data.get('database', {}).get('use', 'sqlite') + self.ap.logger.info(f'Initializing database type: {database_type}...') for manager in database.preregistered_managers: if manager.name == database_type: self.db = manager(self.ap) diff --git a/pkg/persistence/migrations/dbm006_plugin_install_source.py b/pkg/persistence/migrations/dbm006_plugin_install_source.py index 37f74929..7ab0afd3 100644 --- a/pkg/persistence/migrations/dbm006_plugin_install_source.py +++ b/pkg/persistence/migrations/dbm006_plugin_install_source.py @@ -9,9 +9,21 @@ class DBMigratePluginInstallSource(migration.DBMigration): async def upgrade(self): """升级""" # 查询表结构获取所有列名（异步执行 SQL） - result = await self.ap.persistence_mgr.execute_async(sqlalchemy.text('PRAGMA table_info(plugin_settings);')) - # fetchall() 是同步方法，无需 await - columns = [row[1] for row in result.fetchall()] + + columns = [] + + if self.ap.persistence_mgr.db.name == 'postgresql': + result = await self.ap.persistence_mgr.execute_async( + sqlalchemy.text( + "SELECT column_name FROM information_schema.columns WHERE table_name = 'plugin_settings';" + ) + ) + all_result = result.fetchall() + columns = [row[0] for row in all_result] + else: + result = await self.ap.persistence_mgr.execute_async(sqlalchemy.text('PRAGMA table_info(plugin_settings);')) + all_result = result.fetchall() + columns = [row[1] for row in all_result] # 检查并添加 install_source 列 if 'install_source' not in columns: diff --git a/pyproject.toml b/pyproject.toml index e8d8be83..a8ada81e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,7 @@ dependencies = [ "dashscope>=1.23.2", "dingtalk-stream>=0.24.0", "discord-py>=2.5.2", - "pynacl>=1.5.0", # Required for Discord voice support + "pynacl>=1.5.0", # Required for Discord voice support "gewechat-client>=0.1.5", "lark-oapi>=1.4.15", "mcp>=1.8.1", @@ -63,6 +63,7 @@ dependencies = [ "chromadb>=0.4.24", "qdrant-client (>=1.15.1,<2.0.0)", "langbot-plugin==0.1.1b6", + "asyncpg>=0.30.0", ] keywords = [ "bot", diff --git a/templates/config.yaml b/templates/config.yaml index dbb5cc46..00de2a9b 100644 --- a/templates/config.yaml +++ b/templates/config.yaml @@ -21,8 +21,15 @@ system: expire: 604800 secret: '' database: - type: sqlite - engine_url: 'sqlite+aiosqlite:///data/langbot.db' + use: sqlite + sqlite: + path: 'data/langbot.db' + postgresql: + host: '127.0.0.1' + port: 5432 + user: 'postgres' + password: 'postgres' + database: 'postgres' vdb: use: chroma qdrant: