import os
import numpy as np
import mindspore.mindrecord as record

input_ids = [[1,2,3],[4,5,6],[7,8,9]]
intent_ids = [1,2,3]


# 输出的MindSpore Record文件完整路径
MINDRECORD_FILE = "test1.mindrecord"


if os.path.exists(MINDRECORD_FILE):

    os.remove(MINDRECORD_FILE)

    os.remove(MINDRECORD_FILE +".db")



#定义样本数据包含的字段

nlp_schema = {"id": {"type": "int32"}, "input_ids": {"type": "int32","shape":[-1]}, "intent_ids": {"type": "int32"}}



def get_data(input_ids,intent_ids):

    data_list = [ ]

    length = len(intent_ids)

    for i in range(length):

        data_json = {"id": i,

                            "input_ids": np.array(input_ids[i]).astype(np.int32),

                            "intent_ids": np.array(intent_ids[i]).astype(np.int32)}

        data_list.append(data json)

    return data List



#声明MindSpore Record文件格式

writer = record.FileWriter(file_name=MINDRECORD_FILE, shard_num=1)

writer.add schema(nlp_schema,"preprocessed nlp dataset.")

data = get_data(input_ids,intent_ids)

writer.write_raw_data(data)

writer.co㎜it()