华为计算微信公众号
昇腾AI开发者公众号
华为计算微博
华为计算今日头条
class Rerank_Model_Load(nn.Module): def __init__(self, model_name, use_gpu): super(Rerank_Model_Load, self).__init__() self.tokenizer = AutoTokenizer.from_pretrained(os.path.join(model_path, model_name)) if use_gpu: self.model = AutoModelForSequenceClassification.from_pretrained(os.path.join(model_path, model_name)).to("npu:0") else: self.model = AutoModelForSequenceClassification.from_pretrained(os.path.join(model_path, model_name)).to("cpu") self.use_gpu = use_gpu def forward(self, pairs): with torch.no_grad(): if self.use_gpu: inputs = self.tokenizer(pairs, padding=True, truncation=True, return_tensors='pt', max_length=1024).to("npu:0") else: inputs = self.tokenizer(pairs, padding=True, truncation=True, return_tensors='pt', max_length=1024) scores = self.model(**inputs, return_dict=True).logits.view(-1, ).float().cpu().numpy().tolist() # 添加 normalize normalized_scores = [s / max(scores) for s in scores] return normalized_scores
基于这种思路推理重排模型,显卡是用了,但是推理速度太慢了,有木有好的方案,不然没法支撑起rag的运行。求大佬们想想招。
本帖最后由 匿名用户 于 2024/11/22 11:29:10 编辑
我要发帖子
class Rerank_Model_Load(nn.Module): def __init__(self, model_name, use_gpu): super(Rerank_Model_Load, self).__init__() self.tokenizer = AutoTokenizer.from_pretrained(os.path.join(model_path, model_name)) if use_gpu: self.model = AutoModelForSequenceClassification.from_pretrained(os.path.join(model_path, model_name)).to("npu:0") else: self.model = AutoModelForSequenceClassification.from_pretrained(os.path.join(model_path, model_name)).to("cpu") self.use_gpu = use_gpu def forward(self, pairs): with torch.no_grad(): if self.use_gpu: inputs = self.tokenizer(pairs, padding=True, truncation=True, return_tensors='pt', max_length=1024).to("npu:0") else: inputs = self.tokenizer(pairs, padding=True, truncation=True, return_tensors='pt', max_length=1024) scores = self.model(**inputs, return_dict=True).logits.view(-1, ).float().cpu().numpy().tolist() # 添加 normalize normalized_scores = [s / max(scores) for s in scores] return normalized_scores基于这种思路推理重排模型,显卡是用了,但是推理速度太慢了,有木有好的方案,不然没法支撑起rag的运行。求大佬们想想招。