使用MindIE原生接口

文本推理接口

https://{服务IP地址}:{端口号}/infer

POST

curl -H "Accept: application/json" -H "Content-type: application/json" --cacert ca.pem --cert client.pem  --key client.key.pem -X POST -d '{
    "inputs": "My name is Olivier and I",
    "stream": false,
    "parameters": {
        "temperature": 0.5,
        "top_k": 10,
        "top_p": 0.95,
        "max_new_tokens": 20,
        "do_sample": true,
        "seed": null,
        "repetition_penalty": 1.03,
        "details": true,
        "typical_p": 0.5,
        "watermark": false,
        "priority": 5,
        "timeout": 10
    }
}' https://127.0.0.1:1025/infer

{
    "generated_text": "am a French native speaker. I am looking for a job in the hospitality industry. I",
    "details": {
        "finish_reason": "length",
        "generated_tokens": 20,
        "seed": 846930886
    }
}

流式推理接口

https://{服务IP地址}:{端口号}/infer

POST

curl -H "Accept: application/json" -H "Content-type: application/json" --cacert ca.pem --cert client.pem  --key client.key.pem -X POST -d '{
    "inputs": "My name is Olivier and I",
    "stream": true,
    "parameters": {
        "temperature": 0.5,
        "top_k": 10,
        "top_p": 0.95,
        "max_new_tokens": 20,
        "do_sample": true,
        "seed": null,
        "repetition_penalty": 1.03,
        "details": true,
        "typical_p": 0.5,
        "watermark": false,
        "priority": 5,
        "timeout": 10
    }
}' https://127.0.0.1:1025/infer

data: {"prefill_time":45.54,"decode_time":null,"token":{"id":626,"text":"am"}}

data: {"prefill_time":null,"decode_time":128.32,"token":{"id":263,"text":" a"}}

data: {"prefill_time":null,"decode_time":18.17,"token":{"id":5176,"text":" French"}}

data: {"prefill_time":null,"decode_time":16.80,"token":{"id":17739,"text":" photograph"}}

data: {"prefill_time":null,"decode_time":16.80,"token":{"id":261,"text":"er"}}

data: {"prefill_time":null,"decode_time":16.80,"token":{"id":2729,"text":" based"}}

data: {"prefill_time":null,"decode_time":16.80,"token":{"id":297,"text":" in"}}

data: {"prefill_time":null,"decode_time":16.80,"token":{"id":3681,"text":" Paris"}}

data: {"prefill_time":null,"decode_time":16.80,"token":{"id":29889,"text":"."}}

data: {"prefill_time":null,"decode_time":16.80,"token":{"id":13,"text":"\n"}}

data: {"prefill_time":null,"decode_time":16.80,"token":{"id":29902,"text":"I"}}

data: {"prefill_time":null,"decode_time":16.80,"token":{"id":505,"text":" have"}}

data: {"prefill_time":null,"decode_time":16.80,"token":{"id":1063,"text":" been"}}

data: {"prefill_time":null,"decode_time":16.80,"token":{"id":27904,"text":" shooting"}}

data: {"prefill_time":null,"decode_time":16.80,"token":{"id":1951,"text":" since"}}

data: {"prefill_time":null,"decode_time":16.80,"token":{"id":306,"text":" I"}}

data: {"prefill_time":null,"decode_time":16.80,"token":{"id":471,"text":" was"}}

data: {"prefill_time":null,"decode_time":16.80,"token":{"id":29871,"text":" "}}

data: {"prefill_time":null,"decode_time":16.80,"token":{"id":29896,"text":"1"}}

data: {"prefill_time":null,"decode_time":16.80,"generated_text":"am a French photographer based in Paris.\nI have been shooting since I was 15","details":{"finish_reason":"length","generated_tokens":20,"seed":846930886},"token":{"id":29945,"text":null}}