流式文本生成

如创建样例代码 test_generate_stream.py（需要和创建客户端中创建客户端的代码样例utils.py在同一个目录下），并根据实际情况修改model_name，然后执行python test_generate_stream.py命令运行该样例。

import sys
from utils import create_client

if __name__ == "__main__":
    # get argument and create client
    try:
        mindie_client = create_client()
    except Exception as e:
        print("Client Creation falied!")
        sys.exit(1)
    # create input
    prompt = "My name is Olivier and I"
    model_name = "llama_65b" # 需要和服务端配置的modelName保持一致
    parameters = {
        "do_sample": True,
        "temperature": 0.5,
        "top_k": 10,
        "top_p": 0.9,
        "truncate": 5,
        "typical_p": 0.9,
        "seed": 1,
        "repetition_penalty": 1,
        "watermark": True,
        "details": True,
    }
    # apply model inference
    results = mindie_client.generate_stream(
        model_name,
        prompt,
        request_id="1",
        parameters=parameters,
    )
    generated_text = ""
    for cur_res in results:
        if cur_res == "</s>":
            break
        generated_text += cur_res
        print("cur text: %s" % cur_res)
    print("final generated text: %s" % generated_text)