【DeepLink】Profiling收集到打点时序有问题
收藏回复举报
【DeepLink】Profiling收集到打点时序有问题
t('forum.solved') 已解决
发表于2024-02-26 10:59:51
0 查看

基础信息

cann 版本:7.0.0
系统版本:Ubuntu 18.04
kernel版本:5.15.0-91-generic
设备:Atlas 800T A2

问题描述

我们正在开发一个pytorch adaptor模块,仓库地址:https://github.com/DeepLink-org/deeplink.framework。
整体上参考了torch_npu的实现,目前已经完成了对Profiling AscendCL API的对接,对接代码如下:

class AscendProfiler {
public:
  AscendProfiler(const AscendProfiler &) = delete;
  AscendProfiler &operator=(const AscendProfiler &) = delete;

  // AscendProfiler designed as a singleton
  static AscendProfiler& instance();

  void enableProfiler(const std::string &dump_path, bool call_stack);
  void disableProfiler();

  std::unique_ptr<at::ObserverContext> startRecordEvent(const at::RecordFunction &fn);
  void finishRecordEvent(const at::RecordFunction& fn, at::ObserverContext *context);

private:
  AscendProfiler() = default;

private:
  bool enable_ = false;
  aclprofConfig *config_ = nullptr;
  bool call_stack_ = false;
};

AscendProfiler& AscendProfiler::instance() {
  static AscendProfiler profiler;
  return profiler;
}

void AscendProfiler::enableProfiler(const std::string &dump_path, bool call_stack) {
  if (enable_) {
    return;
  }

  call_stack_ = call_stack;
  int32_t device_index = 0;
  DIPU_CALLACLRT(aclrtGetDevice(&device_index));

  uint64_t npu_event = 431;
  uint64_t aicore_metrics = 1;
  static const uint32_t device_num = 1;
  uint32_t device_ids[device_num] = {static_cast<uint32_t>(device_index)};
  aclprofAicoreEvents* events = nullptr;
  config_ = aclprofCreateConfig(
      device_ids, device_num, static_cast<aclprofAicoreMetrics>(aicore_metrics),
      events, npu_event);
  TORCH_CHECK(config_ != nullptr, "aclprofCreateConfig fail, device_index = ", device_index,
      "npu_event = ", npu_event, "aicore_metrics = ", aicore_metrics);

  DIPU_CALLACLRT(aclrtSynchronizeDevice());
  DIPU_CALLACLRT(aclprofInit(dump_path.c_str(), dump_path.size()));
  DIPU_CALLACLRT(aclprofStart(config_));

  at::addThreadLocalCallback(at::RecordFunctionCallback(
    [](const at::RecordFunction& fn) -> std::unique_ptr<at::ObserverContext> {
      return AscendProfiler::instance().startRecordEvent(fn);
    },
    [](const at::RecordFunction& fn, at::ObserverContext* ctx) {
      AscendProfiler::instance().finishRecordEvent(fn, ctx);
    }));
  enable_ = true;
}

void AscendProfiler::disableProfiler() {
  if (!enable_) {
    return;
  }

  DIPU_CALLACLRT(aclrtSynchronizeDevice());
  at::clearThreadLocalCallbacks();
  DIPU_CALLACLRT(aclprofStop(config_));
  DIPU_CALLACLRT(aclprofFinalize());
  enable_ = false;
}

struct AscendObserverContext : public at::ObserverContext {
  AscendObserverContext(void *d, uint32_t n) : data(d), id(n) {}

  void *data = nullptr;
  uint32_t id = 0;
};

std::unique_ptr<at::ObserverContext> AscendProfiler::startRecordEvent(const at::RecordFunction &fn) {
  if (!enable_) {
    return std::unique_ptr<AscendObserverContext>();
  }

  void *stamp = aclprofCreateStamp();
  TORCH_CHECK(stamp != nullptr, "aclprofCreateStamp fail", ", error msg = ", aclGetRecentErrMsg());
  static const std::string tag_name = "torch_op";
  DIPU_CALLACLRT(aclprofSetStampTagName(stamp, tag_name.c_str(), tag_name.size()));
  DIPU_CALLACLRT(aclprofSetStampTraceMessage(stamp, fn.name(), strlen(fn.name())));
  uint32_t range_id = 0;
  DIPU_CALLACLRT(aclprofRangeStart(stamp, &range_id));
  return std::make_unique<AscendObserverContext>(stamp, range_id);
}

void AscendProfiler::finishRecordEvent(const at::RecordFunction& fn, at::ObserverContext* context) {
  if (!enable_) {
    return;
  }

  AscendObserverContext* ctx_ptr = static_cast<AscendObserverContext*>(context);
  DIPU_CALLACLRT(aclprofRangeStop(ctx_ptr->id));
  aclprofDestroyStamp(ctx_ptr->data);
}

void enableProfiler(const std::string &dump_path, bool call_stack) {
  AscendProfiler::instance().enableProfiler(dump_path, call_stack);
}

void disableProfiler() {
  AscendProfiler::instance().disableProfiler();
}

对接完成后,对模型训练进行profiling,使用ui.perfetto.dev查看,发现kernel的时间戳在Node@launch之前,不符合预期;应该是先调用Node@launch来launch kernel,然后才是kernel被调用吧。

profiler.jpg

麻烦华为的专家看看问题出在哪里呢?

我要发帖子