mirror of
https://github.com/langbot-app/LangBot.git
synced 2025-11-25 19:37:36 +08:00
30 lines
550 B
Python
30 lines
550 B
Python
from __future__ import annotations
|
|
|
|
import abc
|
|
import typing
|
|
|
|
from ...core import app
|
|
from .. import entities as llm_entities
|
|
from . import entities
|
|
|
|
|
|
class LLMTokenizer(metaclass=abc.ABCMeta):
|
|
|
|
ap: app.Application
|
|
|
|
def __init__(self, ap: app.Application):
|
|
self.ap = ap
|
|
|
|
async def initialize(self):
|
|
"""初始化分词器
|
|
"""
|
|
pass
|
|
|
|
@abc.abstractmethod
|
|
async def count_token(
|
|
self,
|
|
messages: list[llm_entities.Message],
|
|
model: entities.LLMModelInfo
|
|
) -> int:
|
|
pass
|