流式文本生成

import sys
from utils import create_client
from mindieclient.python.common import Log

logger = Log(__name__).getlog()

if __name__ == "__main__":
    # get argument and create client
    try:
        mindie_client = create_client()
    except Exception as e:
        logger.exception("Client Creation falied!")
        sys.exit(1)
    # create input
    prompt = "My name is Olivier and I"
    model_name = "llama_65b"
    parameters = {
        "do_sample": True,
        "temperature": 0.5,
        "top_k": 10,
        "top_p": 0.9,
        "truncate": 5,
        "typical_p": 0.9,
        "seed": 1,
        "repetition_penalty": 1,
        "watermark": True,
        "details": True,
    }
    # apply model inference
    results = mindie_client.generate_stream(
        model_name,
        prompt,
        request_id="1",
        parameters=parameters,
    )
    generated_text = ""
    for cur_res in results:
        if cur_res == "</s>":
            break
        generated_text += cur_res
        logger.info("cur text: %s", cur_res)
    logger.info("final generated text: %s", generated_text)