query

功能描述

RAG SDK对话功能。

函数原型

def query(text, llm_config, *args, **kwargs)

输入参数说明

返回值说明

调用示例

• 基本对话功能

from paddle.base import libpaddle
from langchain.text_splitter import RecursiveCharacterTextSplitter
from mx_rag.chain import SingleText2TextChain
from mx_rag.document import LoaderMng
from mx_rag.document.loader import DocxLoader, PdfLoader, PowerPointLoader
from mx_rag.embedding.local import TextEmbedding
from mx_rag.embedding.service import TEIEmbedding
from mx_rag.knowledge import KnowledgeDB, KnowledgeStore
from mx_rag.knowledge.handler import upload_files
from mx_rag.knowledge.knowledge import KnowledgeStore
from mx_rag.llm import Text2TextLLM, Img2TextLLM, LLMParameterConfig
from mx_rag.retrievers import Retriever
from mx_rag.storage.document_store import SQLiteDocstore
from mx_rag.storage.vectorstore import MindFAISS
from mx_rag.utils import ClientParam
from mx_rag.llm.llm_parameter import LLMParameterConfig

loader_mng = LoaderMng()
# 加载文档加载器，可以使用mxrag自有的，也可以使用langchain的
loader_mng.register_loader(loader_class=PdfLoader, file_types=[".pdf"])
loader_mng.register_loader(loader_class=DocxLoader, file_types=[".docx"])
loader_mng.register_loader(loader_class=PowerPointLoader, file_types=[".pptx"])
# 加载文档切分器，使用langchain的
loader_mng.register_splitter(splitter_class=RecursiveCharacterTextSplitter,
                             file_types=[".pdf", ".docx", ".txt", ".md", ".xlsx", ".pptx"],
                             splitter_params={"chunk_size": 750,
                                              "chunk_overlap": 150,
                                              "keep_separator": False })

dev = 0
# 加载embedding模型
emb = TextEmbedding("/path/to/acge_text_embedding/", dev_id=dev)
# 初始化向量数据库
vector_store = MindFAISS(x_dim=1024,  devs=[dev],
                                 load_local_index="/path/to/faiss.index",
                                 auto_save=True)
# 初始化文档chunk关系数据库
chunk_store = SQLiteDocstore(db_path="/path/to/sql.db")
# 初始化知识管理关系数据库
knowledge_store = KnowledgeStore(db_path="/path/to/sql.db")
# 添加知识库
knowledge_store.add_knowledge("test", "Default", "admin")
# 初始化知识库管理
knowledge_db = KnowledgeDB(knowledge_store=knowledge_store,
                           chunk_store=chunk_store,
                           vector_store=vector_store,
                           knowledge_name="test",
                           white_paths=["/path/"],
                           user_id="Default"
                           )
# 上传文档到知识库
upload_files(knowledge_db, ["/path/to/file1", "/path/to/file2"], loader_mng, emb.embed_documents, True)
client_param = ClientParam(ca_file="/path/to/ca.crt")
llm = Text2TextLLM(model_name="Meta-Llama-3-8B-Instruct", 
                   base_url="https://x.x.x.x:port/v1/chat/completions", 
                   client_param=client_param)
r = Retriever(vector_store=vector_store, document_store=chunk_store, embed_func=emb.embed_documents, k=1, score_threshold=0.6)
rag = SingleText2TextChain(retriever=r, llm=llm)
response = rag.query("mxVision软件架构包含哪些模块？", LLMParameterConfig(max_tokens=1024, temperature=1.0, top_p=0.1))
print(response)

• 图文并茂对话功能

from paddle.base import libpaddle
from langchain.text_splitter import RecursiveCharacterTextSplitter
from mx_rag.chain import SingleText2TextChain
from mx_rag.document import LoaderMng
from mx_rag.document.loader import DocxLoader, PdfLoader, PowerPointLoader
from mx_rag.embedding.local import TextEmbedding
from mx_rag.embedding.service import TEIEmbedding
from mx_rag.knowledge import KnowledgeDB, KnowledgeStore
from mx_rag.knowledge.handler import upload_files
from mx_rag.knowledge.knowledge import KnowledgeStore
from mx_rag.llm import Text2TextLLM, Img2TextLLM, LLMParameterConfig
from mx_rag.retrievers import Retriever
from mx_rag.storage.document_store import SQLiteDocstore
from mx_rag.storage.vectorstore import MindFAISS
from mx_rag.utils import ClientParam
from mx_rag.llm.llm_parameter import LLMParameterConfig
from typing import List
from langchain_core.documents import Document

# 加载用于解析文档中图片的视觉大模型
vlm = Img2TextLLM(base_url="https://x.x.x.x:port/openai/v1/chat/completions",
                   model_name="Qwen2.5-VL-7B-Instruct",
                   llm_config=LLMParameterConfig(max_tokens=512),
                   client_param=ClientParam(ca_file="/path/to/ca.crt")
                   )
loader_mng = LoaderMng()
# 文档加载器，可以使用mxrag自有的，也可以使用langchain的
loader_mng.register_loader(loader_class=PdfLoader, file_types=[".pdf"], loader_params={"vlm": vlm})
loader_mng.register_loader(loader_class=DocxLoader, file_types=[".docx"], loader_params={"vlm": vlm})
loader_mng.register_loader(loader_class=PowerPointLoader, file_types=[".pptx"], loader_params={"vlm": vlm})
# 加载文档切分器，使用langchain的
loader_mng.register_splitter(splitter_class=RecursiveCharacterTextSplitter,
                             file_types=[".pdf", ".docx", ".txt", ".md", ".xlsx", ".pptx"],
                             splitter_params={"chunk_size": 750,
                                              "chunk_overlap": 150,
                                              "keep_separator": False })

dev = 0
# 加载embedding模型
emb = TextEmbedding("/path/to/acge_text_embedding/", dev_id=dev)
client_param = ClientParam(ca_file="/path/to/ca.crt")
# 初始化向量数据库
vector_store = MindFAISS(x_dim=1024,  devs=[dev],
                                 load_local_index="/path/to/faiss.index",
                                 auto_save=True)
# 初始化文档chunk关系数据库
chunk_store = SQLiteDocstore(db_path="/path/to/sql.db")
# 初始化知识管理关系数据库
knowledge_store = KnowledgeStore(db_path="/path/to/sql.db")
# 添加知识库
knowledge_store.add_knowledge("test", "Default", "admin")
# 初始化知识库管理
knowledge_db = KnowledgeDB(knowledge_store=knowledge_store,
                           chunk_store=chunk_store,
                           vector_store=vector_store,
                           knowledge_name="test",
                           white_paths=["/path/"],
                           user_id="Default"
                           )
# 上传文档到知识库
upload_files(knowledge_db, ["/path/to/file1", "/path/to/file2"], loader_mng, emb.embed_documents, True)
# 定义回调函数，整合问题和检索到的文本与图片描述，生成大模型对话中角色user的content的内容
def user_content_builder(query: str, docs: List[Document], prompt: str=None):
    text_docs = [doc for doc in docs if doc.metadata.get("type", "") == "text"]
    img_docs = [doc for doc in docs if doc.metadata.get("type", "") == "image"]
    user_message = []
    if len(text_docs) > 0:
        # 2. Add text quotes
        user_message.append(f"Text Quotes are:")
        for i, doc in enumerate(text_docs):
            user_message.append(f"\n[{i + 1}] {doc.page_content}")
    if len(img_docs) > 0:
        # 3. Add image quotes vlm-text or ocr-text
        user_message.append("\nImage Quotes are:")
        for i, doc in enumerate(img_docs):
            user_message.append(f"\nimage{i + 1} is described as: {doc.page_content}")
    user_message.append("\n\n")
    # 4. add user question
    user_message.append(f"The user question is: {query}")
    return ''.join(user_message)

# 系统prompt
TEXT_INFER_PROMPT = '''
You are a helpful question-answering assistant. Your task is to generate a interleaved text and image response based on provided questions and quotes. Here‘s how to refine your process:

1. **Evidence Selection**:
   - From both text and image quotes, pinpoint those really relevant for answering the question. Focus on significance and direct relevance.
   - Each image quote is the description of the image.

2. **Answer Construction**:
   - Use Markdown to embed text and images in your response, avoid using obvious headings or divisions; ensure the response flows naturally and cohesively.
   - Conclude with a direct and concise answer to the question in a simple and clear sentence.

3. **Quote Citation**:
   - Cite images using the format `![{conclusion}](image index)`; for the first image, use `![{conclusion}](image1)`;The {conclusion} should be a concise one-sentence summary of the image’s content.
   - Ensure the cite of the image must strict follow `![{conclusion}](image index)`, do not simply stating "See image1", "image1 shows" ,"[image1]" or "image1".
   - Each image or text can only be quoted once.

- Do not cite irrelevant quotes.
- Compose a detailed and articulate interleaved answer to the question.
- Ensure that your answer is logical, informative, and directly ties back to the evidence provided by the quotes.
- If Quote contain text and image, answer must contain both text and image response.
- If Quote only contain text, answer must contain text response, do not contain image.
- Answer in chinese.
'''

client_param = ClientParam(ca_file="/path/to/ca.crt")
# 用于对话的语言大模型
llm = Text2TextLLM(model_name="Meta-Llama-3-8B-Instruct", 
                   base_url="https://x.x.x.x:port/v1/chat/completions", 
                   client_param=client_param)
sys_messages=[{"role": "system", "content": TEXT_INFER_PROMPT}]
r = Retriever(vector_store=vector_store, document_store=chunk_store, embed_func=emb.embed_documents, k=1, score_threshold=0.6)
rag = SingleText2TextChain(retriever=r, llm=llm, sys_messages=sys_messages, user_content_builder=user_content_builder)
response = rag.query("mxVision软件架构包含哪些模块？", LLMParameterConfig(max_tokens=1024, temperature=1.0, top_p=0.1))
# 回答的source_documents中可能会有图片，可在字典metadata中获取图片base64编码
print(response)