From 8b10da04b6fdb428e464ff3ab5f076a8365fb80d Mon Sep 17 00:00:00 2001
From: Luis Vega <vegaluisjose@gmail.com>
Date: Thu, 4 Feb 2021 06:00:03 +0000
Subject: [PATCH 01/10] new experiment

---
 .../backend/contrib/verilator/codegen.cc      |  30 ++--
 .../contrib/verilator/verilator_runtime.cc    | 159 +++++++-----------
 .../contrib/verilator/verilator_runtime.h     |  95 +++++++++++
 3 files changed, 167 insertions(+), 117 deletions(-)
 create mode 100644 src/runtime/contrib/verilator/verilator_runtime.h

diff --git a/src/relay/backend/contrib/verilator/codegen.cc b/src/relay/backend/contrib/verilator/codegen.cc
index 2f61ae540395..aecc0424ea22 100644
--- a/src/relay/backend/contrib/verilator/codegen.cc
+++ b/src/relay/backend/contrib/verilator/codegen.cc
@@ -34,6 +34,7 @@
 #include <sstream>
 
 #include "../../../../runtime/contrib/json/json_node.h"
+#include "../../../../runtime/contrib/verilator/verilator_runtime.h"
 #include "../../utils.h"
 #include "../codegen_json/codegen_json.h"
 
@@ -76,28 +77,28 @@ class VerilatorJSONSerializer : public backend::contrib::JSONSerializer {
 };
 
 /*! \brief Attributes to store the compiler options for Verilator */
-struct VerilatorCompilerConfigNode : public tvm::AttrsNode<VerilatorCompilerConfigNode> {
+struct VerilatorOptionsNode : public tvm::AttrsNode<VerilatorOptionsNode> {
   String lib;
 
-  TVM_DECLARE_ATTRS(VerilatorCompilerConfigNode, "ext.attrs.VerilatorCompilerConfigNode") {
+  TVM_DECLARE_ATTRS(VerilatorOptionsNode, "ext.attrs.VerilatorOptionsNode") {
     TVM_ATTR_FIELD(lib).set_default("libverilator.so");
   }
 };
 
-class VerilatorCompilerConfig : public Attrs {
+class VerilatorOptions : public Attrs {
  public:
-  TVM_DEFINE_NOTNULLABLE_OBJECT_REF_METHODS(VerilatorCompilerConfig, Attrs,
-                                            VerilatorCompilerConfigNode);
+  TVM_DEFINE_NOTNULLABLE_OBJECT_REF_METHODS(VerilatorOptions, Attrs,
+                                            VerilatorOptionsNode);
 };
 
-TVM_REGISTER_NODE_TYPE(VerilatorCompilerConfigNode);
-TVM_REGISTER_PASS_CONFIG_OPTION("relay.ext.verilator.options", VerilatorCompilerConfig);
+TVM_REGISTER_NODE_TYPE(VerilatorOptionsNode);
+TVM_REGISTER_PASS_CONFIG_OPTION("relay.ext.verilator.options", VerilatorOptions);
 
 /*!
  * \brief The external compiler/codegen tool. It takes a Relay expression/module and
  * compile it into a runtime module.
  */
-runtime::Module VerilatorCompiler(const ObjectRef& ref) {
+runtime::Module VerilatorBackend(const ObjectRef& ref) {
   CHECK(ref->IsInstance<FunctionNode>());
   auto func = Downcast<Function>(ref);
   auto func_name = GetExtSymbol(func);
@@ -108,20 +109,19 @@ runtime::Module VerilatorCompiler(const ObjectRef& ref) {
 
   // Get Verilator compiler options
   auto ctx = transform::PassContext::Current();
-  auto cfg = ctx->GetConfig<VerilatorCompilerConfig>("relay.ext.verilator.options");
+  auto cfg = ctx->GetConfig<VerilatorOptions>("relay.ext.verilator.options");
   if (!cfg.defined()) {
-    cfg = AttrsWithDefaultValues<VerilatorCompilerConfig>();
+    cfg = AttrsWithDefaultValues<VerilatorOptions>();
   }
 
   auto lib_name = cfg.value()->lib;
 
-  const auto* pf = runtime::Registry::Get("runtime.verilator_runtime_create");
-  CHECK(pf != nullptr) << "Cannot find JSON runtime module to create";
-  auto mod = (*pf)(lib_name, func_name, graph_json, params);
-  return mod;
+  auto n = make_object<runtime::contrib::VerilatorRuntime>(func_name, graph_json, params);
+  n->LoadLibrary(lib_name);
+  return runtime::Module(n);
 }
 
-TVM_REGISTER_GLOBAL("relay.ext.verilator").set_body_typed(VerilatorCompiler);
+TVM_REGISTER_GLOBAL("relay.ext.verilator").set_body_typed(VerilatorBackend);
 
 }  // namespace contrib
 }  // namespace relay
diff --git a/src/runtime/contrib/verilator/verilator_runtime.cc b/src/runtime/contrib/verilator/verilator_runtime.cc
index 60f36e494da7..52be931c9861 100644
--- a/src/runtime/contrib/verilator/verilator_runtime.cc
+++ b/src/runtime/contrib/verilator/verilator_runtime.cc
@@ -19,7 +19,7 @@
 
 /*!
  * \file src/runtime/contrib/verilator/verilator_runtime.cc
- * \brief A simple JSON runtime for Verilator.
+ * \brief A runtime for Verilator.
  */
 
 #include <dlfcn.h>
@@ -35,130 +35,85 @@
 #include "../json/json_runtime.h"
 #include "verilator_device.h"
 #include "verilator_kernel.h"
+#include "verilator_runtime.h"
 
 namespace tvm {
 namespace runtime {
 namespace contrib {
 
-typedef VerilatorHandle (*VerilatorAllocFunc)();
-typedef void (*VerilatorResetFunc)(VerilatorHandle, int);
-typedef void (*VerilatorAddFunc)(VerilatorHandle, int*, int*, int*, int, int);
-
 using namespace tvm::runtime;
+using namespace tvm::runtime::contrib;
 using namespace tvm::runtime::json;
 
-class VerilatorLibrary : public Library {
- public:
-  ~VerilatorLibrary() {
-    if (lib_handle_) Unload();
-  }
-  void Init(const std::string& name) { Load(name); }
-
-  void* GetSymbol(const char* name) final { return GetSymbol_(name); }
-
- private:
-  // Library handle
-  void* lib_handle_{nullptr};
-  // load the library
-  void Load(const std::string& name) {
-    lib_handle_ = dlopen(name.c_str(), RTLD_LAZY | RTLD_LOCAL);
-    ICHECK(lib_handle_ != nullptr)
-        << "Failed to load dynamic shared library " << name << " " << dlerror();
-  }
+void VerilatorLibrary::Load(const std::string& name) {
+  lib_handle_ = dlopen(name.c_str(), RTLD_LAZY | RTLD_LOCAL);
+  ICHECK(lib_handle_ != nullptr)
+      << "Failed to load dynamic shared library " << name << " " << dlerror();
+}
 
-  void* GetSymbol_(const char* name) { return dlsym(lib_handle_, name); }
+void* VerilatorLibrary::GetSymbol_(const char* name) { return dlsym(lib_handle_, name); }
 
-  void Unload() {
-    dlclose(lib_handle_);
-    lib_handle_ = nullptr;
-  }
-};
-
-class VerilatorJSONRuntime : public JSONRuntimeBase {
- public:
-  VerilatorJSONRuntime(const std::string& symbol_name, const std::string& graph_json,
-                       const Array<String> const_names)
-      : JSONRuntimeBase(symbol_name, graph_json, const_names) {}
+void VerilatorLibrary::Unload() {
+  dlclose(lib_handle_);
+  lib_handle_ = nullptr;
+}
 
-  const char* type_key() const { return "verilator_json"; }
+void VerilatorRuntime::LoadLibrary(const std::string& lib_name) {
+  lib_ = new VerilatorLibrary();
+  lib_->Init(lib_name);
+}
 
-  void LoadLibrary(const std::string& lib_name) {
-    lib_ = new VerilatorLibrary();
-    lib_->Init(lib_name);
-  }
+void VerilatorRuntime::Init(const Array<NDArray>& consts) {
+  // get symbols
+  auto alloc_func = reinterpret_cast<VerilatorAllocFunc>(lib_->GetSymbol("VerilatorAlloc"));
+  ICHECK(alloc_func != nullptr);
+  auto reset_func = reinterpret_cast<VerilatorResetFunc>(lib_->GetSymbol("VerilatorReset"));
+  ICHECK(reset_func != nullptr);
+  vadd_func_ = reinterpret_cast<VerilatorAddFunc>(lib_->GetSymbol("verilator_add"));
+  ICHECK(vadd_func_ != nullptr);
 
-  void Init(const Array<NDArray>& consts) override {
-    // get symbols
-    auto alloc_func = reinterpret_cast<VerilatorAllocFunc>(lib_->GetSymbol("VerilatorAlloc"));
-    ICHECK(alloc_func != nullptr);
-    auto reset_func = reinterpret_cast<VerilatorResetFunc>(lib_->GetSymbol("VerilatorReset"));
-    ICHECK(reset_func != nullptr);
-    vadd_func_ = reinterpret_cast<VerilatorAddFunc>(lib_->GetSymbol("verilator_add"));
-    ICHECK(vadd_func_ != nullptr);
+  // alloc device
+  device_ = (*alloc_func)();
 
-    // alloc device
-    device_ = (*alloc_func)();
+  // reset for 10 cycles
+  (*reset_func)(device_, 10);
 
-    // reset for 10 cycles
-    (*reset_func)(device_, 10);
+  CHECK_EQ(consts.size(), const_idx_.size())
+      << "The number of input constants must match the number of required.";
 
-    CHECK_EQ(consts.size(), const_idx_.size())
-        << "The number of input constants must match the number of required.";
+  // Setup constants entries for weights.
+  SetupConstants(consts);
+}
 
-    // Setup constants entries for weights.
-    SetupConstants(consts);
+void VerilatorRuntime::Run() {
+  std::vector<int*> in_ptr;
+  std::vector<int*> out_ptr;
+  for (size_t i = 0; i < input_nodes_.size(); ++i) {
+    uint32_t eid = EntryID(input_nodes_[i], 0);
+    int* data = static_cast<int*>(data_entry_[eid]->data);
+    in_ptr.push_back(data);
   }
-
-  void Run() override {
-    std::vector<int*> in_ptr;
-    std::vector<int*> out_ptr;
-    for (size_t i = 0; i < input_nodes_.size(); ++i) {
-      uint32_t eid = EntryID(input_nodes_[i], 0);
-      int* data = static_cast<int*>(data_entry_[eid]->data);
-      in_ptr.push_back(data);
-    }
-    for (size_t i = 0; i < outputs_.size(); ++i) {
-      uint32_t eid = EntryID(outputs_[i]);
-      int* data = static_cast<int*>(data_entry_[eid]->data);
-      out_ptr.push_back(data);
-    }
-    for (size_t nid = 0; nid < nodes_.size(); ++nid) {
-      const auto& node = nodes_[nid];
-      if (node.GetOpType() == "kernel") {
-        CHECK_EQ(node.GetOpType(), "kernel");
-        auto op_name = node.GetOpName();
-        if ("add" == op_name) {
-          auto entry = node.GetInputs()[0];
-          auto shape = nodes_[entry.id_].GetOpShape()[entry.index_];
-          (*vadd_func_)(device_, in_ptr[0], in_ptr[1], out_ptr[0], shape[0], shape[1]);
-        } else {
-          LOG(FATAL) << "Unsupported op: " << op_name;
-        }
+  for (size_t i = 0; i < outputs_.size(); ++i) {
+    uint32_t eid = EntryID(outputs_[i]);
+    int* data = static_cast<int*>(data_entry_[eid]->data);
+    out_ptr.push_back(data);
+  }
+  for (size_t nid = 0; nid < nodes_.size(); ++nid) {
+    const auto& node = nodes_[nid];
+    if (node.GetOpType() == "kernel") {
+      CHECK_EQ(node.GetOpType(), "kernel");
+      auto op_name = node.GetOpName();
+      if ("add" == op_name) {
+        auto entry = node.GetInputs()[0];
+        auto shape = nodes_[entry.id_].GetOpShape()[entry.index_];
+        (*vadd_func_)(device_, in_ptr[0], in_ptr[1], out_ptr[0], shape[0], shape[1]);
+      } else {
+        LOG(FATAL) << "Unsupported op: " << op_name;
       }
     }
   }
-
- private:
-  /* The verilator device handle. */
-  VerilatorHandle device_{nullptr};
-  /* The verilator library handle. */
-  VerilatorLibrary* lib_{nullptr};
-  /* The verilator vadd function handle. */
-  VerilatorAddFunc vadd_func_{nullptr};
-};
-
-runtime::Module VerilatorJSONRuntimeCreate(String lib_name, String symbol_name, String graph_json,
-                                           const Array<String>& const_names) {
-  auto n = make_object<VerilatorJSONRuntime>(symbol_name, graph_json, const_names);
-  n->LoadLibrary(lib_name);
-  return runtime::Module(n);
 }
 
-TVM_REGISTER_GLOBAL("runtime.verilator_runtime_create").set_body_typed(VerilatorJSONRuntimeCreate);
-
-TVM_REGISTER_GLOBAL("runtime.module.loadbinary_verilator_json")
-    .set_body_typed(JSONRuntimeBase::LoadFromBinary<VerilatorJSONRuntime>);
-
 }  // namespace contrib
 }  // namespace runtime
 }  // namespace tvm
diff --git a/src/runtime/contrib/verilator/verilator_runtime.h b/src/runtime/contrib/verilator/verilator_runtime.h
new file mode 100644
index 000000000000..44adb9cd9ffd
--- /dev/null
+++ b/src/runtime/contrib/verilator/verilator_runtime.h
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file src/runtime/contrib/verilator/verilator_runtime.h
+ * \brief A runtime for Verilator.
+ */
+
+#include <dlfcn.h>
+#include <tvm/runtime/ndarray.h>
+#include <tvm/runtime/registry.h>
+
+#include <cstddef>
+#include <string>
+#include <vector>
+
+#include "../../library_module.h"
+#include "../json/json_node.h"
+#include "../json/json_runtime.h"
+#include "verilator_device.h"
+#include "verilator_kernel.h"
+
+namespace tvm {
+namespace runtime {
+namespace contrib {
+
+using namespace tvm::runtime;
+using namespace tvm::runtime::contrib;
+using namespace tvm::runtime::json;
+
+typedef VerilatorHandle (*VerilatorAllocFunc)();
+typedef void (*VerilatorResetFunc)(VerilatorHandle, int);
+typedef void (*VerilatorAddFunc)(VerilatorHandle, int*, int*, int*, int, int);
+
+class VerilatorLibrary : public Library {
+ public:
+  ~VerilatorLibrary() {
+    if (lib_handle_) Unload();
+  }
+  void Init(const std::string& name) { Load(name); }
+
+  void* GetSymbol(const char* name) final { return GetSymbol_(name); }
+
+ private:
+  void Load(const std::string& name);
+
+  void* GetSymbol_(const char* name);
+
+  void Unload();
+
+  void* lib_handle_{nullptr};
+};
+
+class VerilatorRuntime : public JSONRuntimeBase {
+ public:
+  VerilatorRuntime(const std::string& symbol_name, const std::string& graph_json,
+                       const Array<String> const_names)
+      : JSONRuntimeBase(symbol_name, graph_json, const_names) {}
+
+  const char* type_key() const { return "verilator"; }
+
+  void LoadLibrary(const std::string& lib_name);
+
+  void Init(const Array<NDArray>& consts) override;
+
+  void Run() override;
+
+ private:
+  /* Device handle */
+  VerilatorHandle device_{nullptr};
+  /* Library handle */
+  VerilatorLibrary* lib_{nullptr};
+  /* Vadd */
+  VerilatorAddFunc vadd_func_{nullptr};
+};
+
+}  // namespace contrib
+}  // namespace runtime
+}  // namespace tvm

From a79b7e8841db46c750593f983be24cbd2d98638e Mon Sep 17 00:00:00 2001
From: Luis Vega <vegaluisjose@gmail.com>
Date: Thu, 4 Feb 2021 17:56:02 +0000
Subject: [PATCH 02/10] save

---
 .../backend/contrib/verilator/codegen.cc      | 35 +++++++++++++++----
 .../contrib/verilator/verilator_runtime.cc    | 18 ++++++++--
 .../contrib/verilator/verilator_runtime.h     | 20 ++++++++---
 .../contrib/test_verilator/infrastructure.py  |  4 ++-
 4 files changed, 62 insertions(+), 15 deletions(-)

diff --git a/src/relay/backend/contrib/verilator/codegen.cc b/src/relay/backend/contrib/verilator/codegen.cc
index aecc0424ea22..bc2d582692a6 100644
--- a/src/relay/backend/contrib/verilator/codegen.cc
+++ b/src/relay/backend/contrib/verilator/codegen.cc
@@ -78,10 +78,24 @@ class VerilatorJSONSerializer : public backend::contrib::JSONSerializer {
 
 /*! \brief Attributes to store the compiler options for Verilator */
 struct VerilatorOptionsNode : public tvm::AttrsNode<VerilatorOptionsNode> {
-  String lib;
+  String lib_path;
+  int reset_cycles;
+  bool profiler_enable;
+  int profiler_cycle_counter_id;
 
   TVM_DECLARE_ATTRS(VerilatorOptionsNode, "ext.attrs.VerilatorOptionsNode") {
-    TVM_ATTR_FIELD(lib).set_default("libverilator.so");
+    TVM_ATTR_FIELD(lib_path)
+      .describe("the design library path")
+      .set_default("libverilator.so");
+    TVM_ATTR_FIELD(reset_cycles)
+      .describe("the number of reset cycles")
+      .set_default(1);
+    TVM_ATTR_FIELD(profiler_enable)
+      .describe("enable profiler")
+      .set_default(false);
+    TVM_ATTR_FIELD(profiler_cycle_counter_id)
+      .describe("profiler cycle counter id")
+      .set_default(0);
   }
 };
 
@@ -95,8 +109,8 @@ TVM_REGISTER_NODE_TYPE(VerilatorOptionsNode);
 TVM_REGISTER_PASS_CONFIG_OPTION("relay.ext.verilator.options", VerilatorOptions);
 
 /*!
- * \brief The external compiler/codegen tool. It takes a Relay expression/module and
- * compile it into a runtime module.
+ * \brief The Verilator codegen tool. It takes a Relay expression/module and
+ * compile it into a Verilator runtime module.
  */
 runtime::Module VerilatorBackend(const ObjectRef& ref) {
   CHECK(ref->IsInstance<FunctionNode>());
@@ -107,6 +121,9 @@ runtime::Module VerilatorBackend(const ObjectRef& ref) {
   std::string graph_json = serializer.GetJSON();
   auto params = serializer.GetParams();
 
+  // Create runtime object
+  auto n = make_object<runtime::contrib::VerilatorRuntime>(func_name, graph_json, params);
+
   // Get Verilator compiler options
   auto ctx = transform::PassContext::Current();
   auto cfg = ctx->GetConfig<VerilatorOptions>("relay.ext.verilator.options");
@@ -114,10 +131,14 @@ runtime::Module VerilatorBackend(const ObjectRef& ref) {
     cfg = AttrsWithDefaultValues<VerilatorOptions>();
   }
 
-  auto lib_name = cfg.value()->lib;
+  n->LoadLibrary(cfg.value()->lib_path);
+  n->SetResetCycles(cfg.value()->reset_cycles);
+
+  if (cfg.value()->profiler_enable) {
+    n->EnableProfiler();
+    n->SetProfilerCycleCounterId(cfg.value()->profiler_cycle_counter_id);
+  }
 
-  auto n = make_object<runtime::contrib::VerilatorRuntime>(func_name, graph_json, params);
-  n->LoadLibrary(lib_name);
   return runtime::Module(n);
 }
 
diff --git a/src/runtime/contrib/verilator/verilator_runtime.cc b/src/runtime/contrib/verilator/verilator_runtime.cc
index 52be931c9861..762921417b0e 100644
--- a/src/runtime/contrib/verilator/verilator_runtime.cc
+++ b/src/runtime/contrib/verilator/verilator_runtime.cc
@@ -63,14 +63,26 @@ void VerilatorRuntime::LoadLibrary(const std::string& lib_name) {
   lib_->Init(lib_name);
 }
 
+void VerilatorRuntime::SetResetCycles(const int cycles) {
+  reset_cycles_ = cycles;
+}
+
+void VerilatorRuntime::EnableProfiler() {
+  prof_enable_ = true;
+}
+
+void VerilatorRuntime::SetProfilerCycleCounterId(const int id) {
+  prof_cycle_counter_id_ = id;
+}
+
 void VerilatorRuntime::Init(const Array<NDArray>& consts) {
   // get symbols
   auto alloc_func = reinterpret_cast<VerilatorAllocFunc>(lib_->GetSymbol("VerilatorAlloc"));
   ICHECK(alloc_func != nullptr);
   auto reset_func = reinterpret_cast<VerilatorResetFunc>(lib_->GetSymbol("VerilatorReset"));
   ICHECK(reset_func != nullptr);
-  vadd_func_ = reinterpret_cast<VerilatorAddFunc>(lib_->GetSymbol("verilator_add"));
-  ICHECK(vadd_func_ != nullptr);
+  add_func_ = reinterpret_cast<VerilatorAddFunc>(lib_->GetSymbol("verilator_add"));
+  ICHECK(add_func_ != nullptr);
 
   // alloc device
   device_ = (*alloc_func)();
@@ -106,7 +118,7 @@ void VerilatorRuntime::Run() {
       if ("add" == op_name) {
         auto entry = node.GetInputs()[0];
         auto shape = nodes_[entry.id_].GetOpShape()[entry.index_];
-        (*vadd_func_)(device_, in_ptr[0], in_ptr[1], out_ptr[0], shape[0], shape[1]);
+        (*add_func_)(device_, in_ptr[0], in_ptr[1], out_ptr[0], shape[0], shape[1]);
       } else {
         LOG(FATAL) << "Unsupported op: " << op_name;
       }
diff --git a/src/runtime/contrib/verilator/verilator_runtime.h b/src/runtime/contrib/verilator/verilator_runtime.h
index 44adb9cd9ffd..b75b8be6d3e5 100644
--- a/src/runtime/contrib/verilator/verilator_runtime.h
+++ b/src/runtime/contrib/verilator/verilator_runtime.h
@@ -77,17 +77,29 @@ class VerilatorRuntime : public JSONRuntimeBase {
 
   void LoadLibrary(const std::string& lib_name);
 
+  void SetResetCycles(const int cycles);
+
+  void EnableProfiler();
+
+  void SetProfilerCycleCounterId(const int id);
+
   void Init(const Array<NDArray>& consts) override;
 
   void Run() override;
 
  private:
-  /* Device handle */
+  /* Device handle. */
   VerilatorHandle device_{nullptr};
-  /* Library handle */
+  /* Library handle. */
   VerilatorLibrary* lib_{nullptr};
-  /* Vadd */
-  VerilatorAddFunc vadd_func_{nullptr};
+  /* Add operator. */
+  VerilatorAddFunc add_func_{nullptr};
+  /* Number of reset cycles. */
+  int reset_cycles_{1};
+  /* Profiler status. */
+  bool prof_enable_{false};
+  /* Profiler cycle counter id. */
+  int prof_cycle_counter_id_{0};
 };
 
 }  // namespace contrib
diff --git a/tests/python/contrib/test_verilator/infrastructure.py b/tests/python/contrib/test_verilator/infrastructure.py
index e8fd943aa8a0..988c9981566e 100644
--- a/tests/python/contrib/test_verilator/infrastructure.py
+++ b/tests/python/contrib/test_verilator/infrastructure.py
@@ -102,8 +102,10 @@ def compile_module(mod):
     if not os.path.isfile(lib):
         compile_hardware()
 
+    opts = {"lib_path": lib, "reset_cycles": 3}
+
     with tvm.transform.PassContext(
-        opt_level=3, config={"relay.ext.verilator.options": {"lib": lib}}
+        opt_level=3, config={"relay.ext.verilator.options": opts}
     ):
         exe = relay.vm.compile(mod, target="llvm", params=None)
         code, lib = exe.save()

From 73ecd5ac0a01f4546e519fa37e62ef9ef6ed8839 Mon Sep 17 00:00:00 2001
From: Luis Vega <vegaluisjose@gmail.com>
Date: Thu, 4 Feb 2021 18:09:15 +0000
Subject: [PATCH 03/10] refactor

---
 .../contrib/verilator/verilator_runtime.cc     | 18 +++++++++---------
 .../contrib/verilator/verilator_runtime.h      |  2 +-
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/src/runtime/contrib/verilator/verilator_runtime.cc b/src/runtime/contrib/verilator/verilator_runtime.cc
index 762921417b0e..ac953f12b9f0 100644
--- a/src/runtime/contrib/verilator/verilator_runtime.cc
+++ b/src/runtime/contrib/verilator/verilator_runtime.cc
@@ -77,18 +77,17 @@ void VerilatorRuntime::SetProfilerCycleCounterId(const int id) {
 
 void VerilatorRuntime::Init(const Array<NDArray>& consts) {
   // get symbols
-  auto alloc_func = reinterpret_cast<VerilatorAllocFunc>(lib_->GetSymbol("VerilatorAlloc"));
-  ICHECK(alloc_func != nullptr);
-  auto reset_func = reinterpret_cast<VerilatorResetFunc>(lib_->GetSymbol("VerilatorReset"));
-  ICHECK(reset_func != nullptr);
-  add_func_ = reinterpret_cast<VerilatorAddFunc>(lib_->GetSymbol("verilator_add"));
-  ICHECK(add_func_ != nullptr);
+  auto alloc = reinterpret_cast<VerilatorAllocFunc>(lib_->GetSymbol("VerilatorAlloc"));
+  ICHECK(alloc != nullptr);
+  auto reset = reinterpret_cast<VerilatorResetFunc>(lib_->GetSymbol("VerilatorReset"));
+  ICHECK(reset != nullptr);
+  add_op_ = reinterpret_cast<VerilatorAddFunc>(lib_->GetSymbol("verilator_add"));
 
   // alloc device
-  device_ = (*alloc_func)();
+  device_ = alloc();
 
   // reset for 10 cycles
-  (*reset_func)(device_, 10);
+  reset(device_, reset_cycles_);
 
   CHECK_EQ(consts.size(), const_idx_.size())
       << "The number of input constants must match the number of required.";
@@ -118,7 +117,8 @@ void VerilatorRuntime::Run() {
       if ("add" == op_name) {
         auto entry = node.GetInputs()[0];
         auto shape = nodes_[entry.id_].GetOpShape()[entry.index_];
-        (*add_func_)(device_, in_ptr[0], in_ptr[1], out_ptr[0], shape[0], shape[1]);
+        ICHECK(add_op_ != nullptr);
+        add_op_(device_, in_ptr[0], in_ptr[1], out_ptr[0], shape[0], shape[1]);
       } else {
         LOG(FATAL) << "Unsupported op: " << op_name;
       }
diff --git a/src/runtime/contrib/verilator/verilator_runtime.h b/src/runtime/contrib/verilator/verilator_runtime.h
index b75b8be6d3e5..54619468c997 100644
--- a/src/runtime/contrib/verilator/verilator_runtime.h
+++ b/src/runtime/contrib/verilator/verilator_runtime.h
@@ -93,7 +93,7 @@ class VerilatorRuntime : public JSONRuntimeBase {
   /* Library handle. */
   VerilatorLibrary* lib_{nullptr};
   /* Add operator. */
-  VerilatorAddFunc add_func_{nullptr};
+  VerilatorAddFunc add_op_{nullptr};
   /* Number of reset cycles. */
   int reset_cycles_{1};
   /* Profiler status. */

From 6b357acc98bff1f5fa1cae9833da41586f6e4e00 Mon Sep 17 00:00:00 2001
From: Luis Vega <vegaluisjose@gmail.com>
Date: Thu, 4 Feb 2021 18:17:36 +0000
Subject: [PATCH 04/10] refactor library

---
 .../contrib/verilator/verilator_runtime.cc        | 15 +++++++++------
 src/runtime/contrib/verilator/verilator_runtime.h | 14 ++++----------
 2 files changed, 13 insertions(+), 16 deletions(-)

diff --git a/src/runtime/contrib/verilator/verilator_runtime.cc b/src/runtime/contrib/verilator/verilator_runtime.cc
index ac953f12b9f0..f3cfe4ad0104 100644
--- a/src/runtime/contrib/verilator/verilator_runtime.cc
+++ b/src/runtime/contrib/verilator/verilator_runtime.cc
@@ -45,22 +45,25 @@ using namespace tvm::runtime;
 using namespace tvm::runtime::contrib;
 using namespace tvm::runtime::json;
 
+VerilatorLibrary::~VerilatorLibrary() {
+  if (lib_handle_) {
+    dlclose(lib_handle_);
+    lib_handle_ = nullptr;
+  }
+}
+
 void VerilatorLibrary::Load(const std::string& name) {
   lib_handle_ = dlopen(name.c_str(), RTLD_LAZY | RTLD_LOCAL);
   ICHECK(lib_handle_ != nullptr)
       << "Failed to load dynamic shared library " << name << " " << dlerror();
 }
 
-void* VerilatorLibrary::GetSymbol_(const char* name) { return dlsym(lib_handle_, name); }
+void* VerilatorLibrary::GetSymbol(const char* name) { return dlsym(lib_handle_, name); }
 
-void VerilatorLibrary::Unload() {
-  dlclose(lib_handle_);
-  lib_handle_ = nullptr;
-}
 
 void VerilatorRuntime::LoadLibrary(const std::string& lib_name) {
   lib_ = new VerilatorLibrary();
-  lib_->Init(lib_name);
+  lib_->Load(lib_name);
 }
 
 void VerilatorRuntime::SetResetCycles(const int cycles) {
diff --git a/src/runtime/contrib/verilator/verilator_runtime.h b/src/runtime/contrib/verilator/verilator_runtime.h
index 54619468c997..e2b098530529 100644
--- a/src/runtime/contrib/verilator/verilator_runtime.h
+++ b/src/runtime/contrib/verilator/verilator_runtime.h
@@ -50,20 +50,14 @@ typedef void (*VerilatorAddFunc)(VerilatorHandle, int*, int*, int*, int, int);
 
 class VerilatorLibrary : public Library {
  public:
-  ~VerilatorLibrary() {
-    if (lib_handle_) Unload();
-  }
-  void Init(const std::string& name) { Load(name); }
+  ~VerilatorLibrary();
 
-  void* GetSymbol(const char* name) final { return GetSymbol_(name); }
-
- private:
   void Load(const std::string& name);
 
-  void* GetSymbol_(const char* name);
-
-  void Unload();
+  void* GetSymbol(const char* name) final;
 
+ private:
+  /* Library handle. */
   void* lib_handle_{nullptr};
 };
 

From 7605d69c32a59cbcda0deee7d735e5c01644205e Mon Sep 17 00:00:00 2001
From: Luis Vega <vegaluisjose@gmail.com>
Date: Thu, 4 Feb 2021 19:15:31 +0000
Subject: [PATCH 05/10] add profiler

---
 .../contrib/verilator/verilator_runtime.cc    | 30 +++++++++++++++++--
 .../contrib/verilator/verilator_runtime.h     | 14 +++++++++
 2 files changed, 42 insertions(+), 2 deletions(-)

diff --git a/src/runtime/contrib/verilator/verilator_runtime.cc b/src/runtime/contrib/verilator/verilator_runtime.cc
index f3cfe4ad0104..71bab66ac844 100644
--- a/src/runtime/contrib/verilator/verilator_runtime.cc
+++ b/src/runtime/contrib/verilator/verilator_runtime.cc
@@ -60,6 +60,22 @@ void VerilatorLibrary::Load(const std::string& name) {
 
 void* VerilatorLibrary::GetSymbol(const char* name) { return dlsym(lib_handle_, name); }
 
+void VerilatorProfiler::Clear() {
+  cycle_counter = 0;
+}
+
+std::string VerilatorProfiler::AsJSON() {
+  std::ostringstream os;
+  os << "{\n"
+     << " \"cycle_counter\":" << cycle_counter << "\n"
+     <<"}\n";
+  return os.str();
+}
+
+VerilatorProfiler* VerilatorProfiler::ThreadLocal() {
+  static thread_local VerilatorProfiler inst;
+  return &inst;
+}
 
 void VerilatorRuntime::LoadLibrary(const std::string& lib_name) {
   lib_ = new VerilatorLibrary();
@@ -86,10 +102,10 @@ void VerilatorRuntime::Init(const Array<NDArray>& consts) {
   ICHECK(reset != nullptr);
   add_op_ = reinterpret_cast<VerilatorAddFunc>(lib_->GetSymbol("verilator_add"));
 
-  // alloc device
+  // alloc verilator device
   device_ = alloc();
 
-  // reset for 10 cycles
+  // reset verilator device
   reset(device_, reset_cycles_);
 
   CHECK_EQ(consts.size(), const_idx_.size())
@@ -129,6 +145,16 @@ void VerilatorRuntime::Run() {
   }
 }
 
+TVM_REGISTER_GLOBAL("verilator.profiler_clear")
+.set_body([](TVMArgs args, TVMRetValue* rv) {
+    VerilatorProfiler::ThreadLocal()->Clear();
+  });
+
+TVM_REGISTER_GLOBAL("verilator.profiler_status")
+.set_body([](TVMArgs args, TVMRetValue* rv) {
+    *rv = VerilatorProfiler::ThreadLocal()->AsJSON();
+  });
+
 }  // namespace contrib
 }  // namespace runtime
 }  // namespace tvm
diff --git a/src/runtime/contrib/verilator/verilator_runtime.h b/src/runtime/contrib/verilator/verilator_runtime.h
index e2b098530529..9b595c9840d7 100644
--- a/src/runtime/contrib/verilator/verilator_runtime.h
+++ b/src/runtime/contrib/verilator/verilator_runtime.h
@@ -61,6 +61,20 @@ class VerilatorLibrary : public Library {
   void* lib_handle_{nullptr};
 };
 
+class VerilatorProfiler {
+ public:
+  /*! \brief clear the profiler */
+  void Clear();
+
+  std::string AsJSON();
+
+  static VerilatorProfiler* ThreadLocal();
+
+ private:
+  /*! \brief the number of cycle counter */
+  uint32_t cycle_counter{0};
+};
+
 class VerilatorRuntime : public JSONRuntimeBase {
  public:
   VerilatorRuntime(const std::string& symbol_name, const std::string& graph_json,

From cbf174635e97e694813cbf005e50187e3542fda0 Mon Sep 17 00:00:00 2001
From: Luis Vega <vegaluisjose@gmail.com>
Date: Thu, 4 Feb 2021 19:59:53 +0000
Subject: [PATCH 06/10] refactor

---
 src/runtime/contrib/verilator/verilator_runtime.cc    |  9 +++++++++
 src/runtime/contrib/verilator/verilator_runtime.h     | 10 ++++++----
 tests/python/contrib/test_verilator/infrastructure.py |  2 +-
 3 files changed, 16 insertions(+), 5 deletions(-)

diff --git a/src/runtime/contrib/verilator/verilator_runtime.cc b/src/runtime/contrib/verilator/verilator_runtime.cc
index 71bab66ac844..6ae8a5547565 100644
--- a/src/runtime/contrib/verilator/verilator_runtime.cc
+++ b/src/runtime/contrib/verilator/verilator_runtime.cc
@@ -100,11 +100,16 @@ void VerilatorRuntime::Init(const Array<NDArray>& consts) {
   ICHECK(alloc != nullptr);
   auto reset = reinterpret_cast<VerilatorResetFunc>(lib_->GetSymbol("VerilatorReset"));
   ICHECK(reset != nullptr);
+  read_ = reinterpret_cast<VerilatorReadFunc>(lib_->GetSymbol("VerilatorRead"));
+  ICHECK(read_ != nullptr);
   add_op_ = reinterpret_cast<VerilatorAddFunc>(lib_->GetSymbol("verilator_add"));
 
   // alloc verilator device
   device_ = alloc();
 
+  // enable profiler
+  if (prof_enable_) prof_ = VerilatorProfiler::ThreadLocal();
+
   // reset verilator device
   reset(device_, reset_cycles_);
 
@@ -143,6 +148,10 @@ void VerilatorRuntime::Run() {
       }
     }
   }
+  if (prof_enable_) {
+    int cycles = read_(device_, prof_cycle_counter_id_, 0);
+    prof_->cycle_counter += cycles;
+  }
 }
 
 TVM_REGISTER_GLOBAL("verilator.profiler_clear")
diff --git a/src/runtime/contrib/verilator/verilator_runtime.h b/src/runtime/contrib/verilator/verilator_runtime.h
index 9b595c9840d7..db31ae1aae7a 100644
--- a/src/runtime/contrib/verilator/verilator_runtime.h
+++ b/src/runtime/contrib/verilator/verilator_runtime.h
@@ -47,6 +47,7 @@ using namespace tvm::runtime::json;
 typedef VerilatorHandle (*VerilatorAllocFunc)();
 typedef void (*VerilatorResetFunc)(VerilatorHandle, int);
 typedef void (*VerilatorAddFunc)(VerilatorHandle, int*, int*, int*, int, int);
+typedef int (*VerilatorReadFunc)(VerilatorHandle, int, int);
 
 class VerilatorLibrary : public Library {
  public:
@@ -63,16 +64,15 @@ class VerilatorLibrary : public Library {
 
 class VerilatorProfiler {
  public:
+  /*! \brief the number of cycle counter */
+  uint32_t cycle_counter{0};
+
   /*! \brief clear the profiler */
   void Clear();
 
   std::string AsJSON();
 
   static VerilatorProfiler* ThreadLocal();
-
- private:
-  /*! \brief the number of cycle counter */
-  uint32_t cycle_counter{0};
 };
 
 class VerilatorRuntime : public JSONRuntimeBase {
@@ -100,6 +100,8 @@ class VerilatorRuntime : public JSONRuntimeBase {
   VerilatorHandle device_{nullptr};
   /* Library handle. */
   VerilatorLibrary* lib_{nullptr};
+  VerilatorProfiler* prof_{nullptr};
+  VerilatorReadFunc read_{nullptr};
   /* Add operator. */
   VerilatorAddFunc add_op_{nullptr};
   /* Number of reset cycles. */
diff --git a/tests/python/contrib/test_verilator/infrastructure.py b/tests/python/contrib/test_verilator/infrastructure.py
index 988c9981566e..2f49cad4d8cf 100644
--- a/tests/python/contrib/test_verilator/infrastructure.py
+++ b/tests/python/contrib/test_verilator/infrastructure.py
@@ -102,7 +102,7 @@ def compile_module(mod):
     if not os.path.isfile(lib):
         compile_hardware()
 
-    opts = {"lib_path": lib, "reset_cycles": 3}
+    opts = {"lib_path": lib}
 
     with tvm.transform.PassContext(
         opt_level=3, config={"relay.ext.verilator.options": opts}

From 20179b9533758fa87e40347034d7a3b34fc0cf6f Mon Sep 17 00:00:00 2001
From: Luis Vega <vegaluisjose@gmail.com>
Date: Thu, 4 Feb 2021 20:08:29 +0000
Subject: [PATCH 07/10] refactor

---
 .../backend/contrib/verilator/codegen.cc      | 19 +++------
 .../contrib/verilator/verilator_runtime.cc    | 39 +++++++------------
 .../contrib/verilator/verilator_runtime.h     |  6 ++-
 .../contrib/test_verilator/infrastructure.py  |  4 +-
 4 files changed, 26 insertions(+), 42 deletions(-)

diff --git a/src/relay/backend/contrib/verilator/codegen.cc b/src/relay/backend/contrib/verilator/codegen.cc
index bc2d582692a6..13cd488efad5 100644
--- a/src/relay/backend/contrib/verilator/codegen.cc
+++ b/src/relay/backend/contrib/verilator/codegen.cc
@@ -84,25 +84,16 @@ struct VerilatorOptionsNode : public tvm::AttrsNode<VerilatorOptionsNode> {
   int profiler_cycle_counter_id;
 
   TVM_DECLARE_ATTRS(VerilatorOptionsNode, "ext.attrs.VerilatorOptionsNode") {
-    TVM_ATTR_FIELD(lib_path)
-      .describe("the design library path")
-      .set_default("libverilator.so");
-    TVM_ATTR_FIELD(reset_cycles)
-      .describe("the number of reset cycles")
-      .set_default(1);
-    TVM_ATTR_FIELD(profiler_enable)
-      .describe("enable profiler")
-      .set_default(false);
-    TVM_ATTR_FIELD(profiler_cycle_counter_id)
-      .describe("profiler cycle counter id")
-      .set_default(0);
+    TVM_ATTR_FIELD(lib_path).describe("the design library path").set_default("libverilator.so");
+    TVM_ATTR_FIELD(reset_cycles).describe("the number of reset cycles").set_default(1);
+    TVM_ATTR_FIELD(profiler_enable).describe("enable profiler").set_default(false);
+    TVM_ATTR_FIELD(profiler_cycle_counter_id).describe("profiler cycle counter id").set_default(0);
   }
 };
 
 class VerilatorOptions : public Attrs {
  public:
-  TVM_DEFINE_NOTNULLABLE_OBJECT_REF_METHODS(VerilatorOptions, Attrs,
-                                            VerilatorOptionsNode);
+  TVM_DEFINE_NOTNULLABLE_OBJECT_REF_METHODS(VerilatorOptions, Attrs, VerilatorOptionsNode);
 };
 
 TVM_REGISTER_NODE_TYPE(VerilatorOptionsNode);
diff --git a/src/runtime/contrib/verilator/verilator_runtime.cc b/src/runtime/contrib/verilator/verilator_runtime.cc
index 6ae8a5547565..49bfcd21ec0c 100644
--- a/src/runtime/contrib/verilator/verilator_runtime.cc
+++ b/src/runtime/contrib/verilator/verilator_runtime.cc
@@ -22,6 +22,8 @@
  * \brief A runtime for Verilator.
  */
 
+#include "verilator_runtime.h"
+
 #include <dlfcn.h>
 #include <tvm/runtime/ndarray.h>
 #include <tvm/runtime/registry.h>
@@ -35,7 +37,6 @@
 #include "../json/json_runtime.h"
 #include "verilator_device.h"
 #include "verilator_kernel.h"
-#include "verilator_runtime.h"
 
 namespace tvm {
 namespace runtime {
@@ -54,21 +55,19 @@ VerilatorLibrary::~VerilatorLibrary() {
 
 void VerilatorLibrary::Load(const std::string& name) {
   lib_handle_ = dlopen(name.c_str(), RTLD_LAZY | RTLD_LOCAL);
-  ICHECK(lib_handle_ != nullptr)
-      << "Failed to load dynamic shared library " << name << " " << dlerror();
+  ICHECK(lib_handle_ != nullptr) << "Failed to load dynamic shared library " << name << " "
+                                 << dlerror();
 }
 
 void* VerilatorLibrary::GetSymbol(const char* name) { return dlsym(lib_handle_, name); }
 
-void VerilatorProfiler::Clear() {
-  cycle_counter = 0;
-}
+void VerilatorProfiler::Clear() { cycle_counter = 0; }
 
 std::string VerilatorProfiler::AsJSON() {
   std::ostringstream os;
   os << "{\n"
      << " \"cycle_counter\":" << cycle_counter << "\n"
-     <<"}\n";
+     << "}\n";
   return os.str();
 }
 
@@ -82,17 +81,11 @@ void VerilatorRuntime::LoadLibrary(const std::string& lib_name) {
   lib_->Load(lib_name);
 }
 
-void VerilatorRuntime::SetResetCycles(const int cycles) {
-  reset_cycles_ = cycles;
-}
+void VerilatorRuntime::SetResetCycles(const int cycles) { reset_cycles_ = cycles; }
 
-void VerilatorRuntime::EnableProfiler() {
-  prof_enable_ = true;
-}
+void VerilatorRuntime::EnableProfiler() { prof_enable_ = true; }
 
-void VerilatorRuntime::SetProfilerCycleCounterId(const int id) {
-  prof_cycle_counter_id_ = id;
-}
+void VerilatorRuntime::SetProfilerCycleCounterId(const int id) { prof_cycle_counter_id_ = id; }
 
 void VerilatorRuntime::Init(const Array<NDArray>& consts) {
   // get symbols
@@ -154,15 +147,13 @@ void VerilatorRuntime::Run() {
   }
 }
 
-TVM_REGISTER_GLOBAL("verilator.profiler_clear")
-.set_body([](TVMArgs args, TVMRetValue* rv) {
-    VerilatorProfiler::ThreadLocal()->Clear();
-  });
+TVM_REGISTER_GLOBAL("verilator.profiler_clear").set_body([](TVMArgs args, TVMRetValue* rv) {
+  VerilatorProfiler::ThreadLocal()->Clear();
+});
 
-TVM_REGISTER_GLOBAL("verilator.profiler_status")
-.set_body([](TVMArgs args, TVMRetValue* rv) {
-    *rv = VerilatorProfiler::ThreadLocal()->AsJSON();
-  });
+TVM_REGISTER_GLOBAL("verilator.profiler_status").set_body([](TVMArgs args, TVMRetValue* rv) {
+  *rv = VerilatorProfiler::ThreadLocal()->AsJSON();
+});
 
 }  // namespace contrib
 }  // namespace runtime
diff --git a/src/runtime/contrib/verilator/verilator_runtime.h b/src/runtime/contrib/verilator/verilator_runtime.h
index db31ae1aae7a..1dc8b625dba1 100644
--- a/src/runtime/contrib/verilator/verilator_runtime.h
+++ b/src/runtime/contrib/verilator/verilator_runtime.h
@@ -22,6 +22,9 @@
  * \brief A runtime for Verilator.
  */
 
+#ifndef TVM_RUNTIME_CONTRIB_VERILATOR_VERILATOR_RUNTIME_H_
+#define TVM_RUNTIME_CONTRIB_VERILATOR_VERILATOR_RUNTIME_H_
+
 #include <dlfcn.h>
 #include <tvm/runtime/ndarray.h>
 #include <tvm/runtime/registry.h>
@@ -78,7 +81,7 @@ class VerilatorProfiler {
 class VerilatorRuntime : public JSONRuntimeBase {
  public:
   VerilatorRuntime(const std::string& symbol_name, const std::string& graph_json,
-                       const Array<String> const_names)
+                   const Array<String> const_names)
       : JSONRuntimeBase(symbol_name, graph_json, const_names) {}
 
   const char* type_key() const { return "verilator"; }
@@ -115,3 +118,4 @@ class VerilatorRuntime : public JSONRuntimeBase {
 }  // namespace contrib
 }  // namespace runtime
 }  // namespace tvm
+#endif  // TVM_RUNTIME_CONTRIB_VERILATOR_VERILATOR_RUNTIME_H_
diff --git a/tests/python/contrib/test_verilator/infrastructure.py b/tests/python/contrib/test_verilator/infrastructure.py
index 2f49cad4d8cf..7e4c297853d5 100644
--- a/tests/python/contrib/test_verilator/infrastructure.py
+++ b/tests/python/contrib/test_verilator/infrastructure.py
@@ -104,9 +104,7 @@ def compile_module(mod):
 
     opts = {"lib_path": lib}
 
-    with tvm.transform.PassContext(
-        opt_level=3, config={"relay.ext.verilator.options": opts}
-    ):
+    with tvm.transform.PassContext(opt_level=3, config={"relay.ext.verilator.options": opts}):
         exe = relay.vm.compile(mod, target="llvm", params=None)
         code, lib = exe.save()
         return runtime.vm.Executable.load_exec(code, lib)

From 7e7424c56d9498eff95bdc5898bf507da73f49e2 Mon Sep 17 00:00:00 2001
From: Luis Vega <vegaluisjose@gmail.com>
Date: Thu, 4 Feb 2021 20:51:02 +0000
Subject: [PATCH 08/10] add docs

---
 .../contrib/verilator/verilator_device.h      | 39 ++++++++++++++++---
 .../contrib/verilator/verilator_runtime.h     | 26 +++++++++----
 2 files changed, 52 insertions(+), 13 deletions(-)

diff --git a/src/runtime/contrib/verilator/verilator_device.h b/src/runtime/contrib/verilator/verilator_device.h
index acd91a53bcff..298e41c06daf 100644
--- a/src/runtime/contrib/verilator/verilator_device.h
+++ b/src/runtime/contrib/verilator/verilator_device.h
@@ -31,24 +31,51 @@ namespace tvm {
 namespace runtime {
 namespace contrib {
 
+/*! \brief Verilator device resource context  */
 typedef void* VerilatorHandle;
 
-/* allocate Verilator object */
+/*!
+ * \brief Allocate a verilator device resource handle
+ * \return The verilator device handle.
+ */
 extern "C" TVM_DLL VerilatorHandle VerilatorAlloc();
 
-/* deallocate Verilator object */
+/*!
+ * \brief Free a verilator device handle
+ * \param handle The verilator device handle to be freed.
+ */
 extern "C" TVM_DLL void VerilatorDealloc(VerilatorHandle handle);
 
-/* read Verilator register or memory */
+/*!
+ * \brief Read verilator register or memory
+ * \param handle The verilator device handle.
+ * \param id The register or memory identifier.
+ * \param addr The register or memory address (word-level).
+ * \return The value of register or memory.
+ */
 extern "C" TVM_DLL int VerilatorRead(VerilatorHandle handle, int id, int addr);
 
-/* write Verilator register or memory */
+/*!
+ * \brief Write verilator register or memory
+ * \param handle The verilator device handle.
+ * \param id The register or memory identifier.
+ * \param addr The register or memory address (word-level).
+ * \param value The value of register or memory.
+ */
 extern "C" TVM_DLL void VerilatorWrite(VerilatorHandle handle, int id, int addr, int value);
 
-/* reset Verilator for n clock cycles */
+/*!
+ * \brief Reset Verilator for n clock cycles
+ * \param handle The verilator device handle.
+ * \param n The number of reset cycles.
+ */
 extern "C" TVM_DLL void VerilatorReset(VerilatorHandle handle, int n);
 
-/* run Verilator for n clock cycles */
+/*!
+ * \brief Run Verilator for n clock cycles
+ * \param handle The verilator device handle.
+ * \param n The number of run cycles.
+ */
 extern "C" TVM_DLL void VerilatorRun(VerilatorHandle handle, int n);
 
 }  // namespace contrib
diff --git a/src/runtime/contrib/verilator/verilator_runtime.h b/src/runtime/contrib/verilator/verilator_runtime.h
index 1dc8b625dba1..48308933e03e 100644
--- a/src/runtime/contrib/verilator/verilator_runtime.h
+++ b/src/runtime/contrib/verilator/verilator_runtime.h
@@ -56,12 +56,14 @@ class VerilatorLibrary : public Library {
  public:
   ~VerilatorLibrary();
 
+  /*! \brief load library */
   void Load(const std::string& name);
 
+  /*! \brief get symbol from libray */
   void* GetSymbol(const char* name) final;
 
  private:
-  /* Library handle. */
+  /*! \brief the library handle */
   void* lib_handle_{nullptr};
 };
 
@@ -73,8 +75,10 @@ class VerilatorProfiler {
   /*! \brief clear the profiler */
   void Clear();
 
+  /*! \brief get profiler data */
   std::string AsJSON();
 
+  /*! \brief profiler constructor */
   static VerilatorProfiler* ThreadLocal();
 };
 
@@ -86,32 +90,40 @@ class VerilatorRuntime : public JSONRuntimeBase {
 
   const char* type_key() const { return "verilator"; }
 
+  /*! \brief load verilator library */
   void LoadLibrary(const std::string& lib_name);
 
+  /*! \brief set the number of reset cycles */
   void SetResetCycles(const int cycles);
 
+  /*! \brief enable profiler */
   void EnableProfiler();
 
+  /*! \brief set cycle counter register id */
   void SetProfilerCycleCounterId(const int id);
 
+  /*! \brief init verilator runtime */
   void Init(const Array<NDArray>& consts) override;
 
+  /*! \brief run verilator runtime */
   void Run() override;
 
  private:
-  /* Device handle. */
+  /*! \brief the verilator device */
   VerilatorHandle device_{nullptr};
-  /* Library handle. */
+  /*! \brief the verilator library */
   VerilatorLibrary* lib_{nullptr};
+  /*! \brief the verilator profiler */
   VerilatorProfiler* prof_{nullptr};
+  /*! \brief the verilator read function */
   VerilatorReadFunc read_{nullptr};
-  /* Add operator. */
+  /*! \brief the verilator add op function */
   VerilatorAddFunc add_op_{nullptr};
-  /* Number of reset cycles. */
+  /*! \brief the verilator reset cycles */
   int reset_cycles_{1};
-  /* Profiler status. */
+  /*! \brief the verilator profiler status */
   bool prof_enable_{false};
-  /* Profiler cycle counter id. */
+  /*! \brief the verilator profiler cycle counter id */
   int prof_cycle_counter_id_{0};
 };
 

From 4f8505e580cbbec6166fecc02a8c8acc6ac74257 Mon Sep 17 00:00:00 2001
From: Luis Vega <vegaluisjose@gmail.com>
Date: Fri, 5 Feb 2021 17:44:47 +0000
Subject: [PATCH 09/10] update comment

---
 src/relay/backend/contrib/verilator/codegen.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/relay/backend/contrib/verilator/codegen.cc b/src/relay/backend/contrib/verilator/codegen.cc
index 13cd488efad5..78c6429311ad 100644
--- a/src/relay/backend/contrib/verilator/codegen.cc
+++ b/src/relay/backend/contrib/verilator/codegen.cc
@@ -76,7 +76,7 @@ class VerilatorJSONSerializer : public backend::contrib::JSONSerializer {
   }
 };
 
-/*! \brief Attributes to store the compiler options for Verilator */
+/*! \brief Attributes to store options for Verilator */
 struct VerilatorOptionsNode : public tvm::AttrsNode<VerilatorOptionsNode> {
   String lib_path;
   int reset_cycles;

From 80c67a9bad1b22f512f9e589c3c1523119307c5b Mon Sep 17 00:00:00 2001
From: Luis Vega <vegaluisjose@gmail.com>
Date: Sun, 14 Feb 2021 20:15:37 +0000
Subject: [PATCH 10/10] add deallocator

---
 src/relay/backend/contrib/verilator/codegen.cc     |  2 +-
 src/runtime/contrib/verilator/verilator_runtime.cc | 13 +++++++++----
 src/runtime/contrib/verilator/verilator_runtime.h  |  9 +++++++--
 3 files changed, 17 insertions(+), 7 deletions(-)

diff --git a/src/relay/backend/contrib/verilator/codegen.cc b/src/relay/backend/contrib/verilator/codegen.cc
index 78c6429311ad..b206288f7e96 100644
--- a/src/relay/backend/contrib/verilator/codegen.cc
+++ b/src/relay/backend/contrib/verilator/codegen.cc
@@ -122,7 +122,7 @@ runtime::Module VerilatorBackend(const ObjectRef& ref) {
     cfg = AttrsWithDefaultValues<VerilatorOptions>();
   }
 
-  n->LoadLibrary(cfg.value()->lib_path);
+  n->SetLibrary(cfg.value()->lib_path);
   n->SetResetCycles(cfg.value()->reset_cycles);
 
   if (cfg.value()->profiler_enable) {
diff --git a/src/runtime/contrib/verilator/verilator_runtime.cc b/src/runtime/contrib/verilator/verilator_runtime.cc
index 49bfcd21ec0c..bc96b69f2ffe 100644
--- a/src/runtime/contrib/verilator/verilator_runtime.cc
+++ b/src/runtime/contrib/verilator/verilator_runtime.cc
@@ -76,11 +76,15 @@ VerilatorProfiler* VerilatorProfiler::ThreadLocal() {
   return &inst;
 }
 
-void VerilatorRuntime::LoadLibrary(const std::string& lib_name) {
-  lib_ = new VerilatorLibrary();
-  lib_->Load(lib_name);
+VerilatorRuntime::~VerilatorRuntime() {
+  auto dealloc = reinterpret_cast<VerilatorDeallocFunc>(lib_->GetSymbol("VerilatorDealloc"));
+  ICHECK(dealloc != nullptr);
+  dealloc(device_);
+  lib_->~VerilatorLibrary();
 }
 
+void VerilatorRuntime::SetLibrary(const std::string& lib_path) { lib_path_ = lib_path; }
+
 void VerilatorRuntime::SetResetCycles(const int cycles) { reset_cycles_ = cycles; }
 
 void VerilatorRuntime::EnableProfiler() { prof_enable_ = true; }
@@ -88,7 +92,8 @@ void VerilatorRuntime::EnableProfiler() { prof_enable_ = true; }
 void VerilatorRuntime::SetProfilerCycleCounterId(const int id) { prof_cycle_counter_id_ = id; }
 
 void VerilatorRuntime::Init(const Array<NDArray>& consts) {
-  // get symbols
+  lib_ = new VerilatorLibrary();
+  lib_->Load(lib_path_);
   auto alloc = reinterpret_cast<VerilatorAllocFunc>(lib_->GetSymbol("VerilatorAlloc"));
   ICHECK(alloc != nullptr);
   auto reset = reinterpret_cast<VerilatorResetFunc>(lib_->GetSymbol("VerilatorReset"));
diff --git a/src/runtime/contrib/verilator/verilator_runtime.h b/src/runtime/contrib/verilator/verilator_runtime.h
index 48308933e03e..acdaa3b03ce2 100644
--- a/src/runtime/contrib/verilator/verilator_runtime.h
+++ b/src/runtime/contrib/verilator/verilator_runtime.h
@@ -48,6 +48,7 @@ using namespace tvm::runtime::contrib;
 using namespace tvm::runtime::json;
 
 typedef VerilatorHandle (*VerilatorAllocFunc)();
+typedef void (*VerilatorDeallocFunc)(VerilatorHandle);
 typedef void (*VerilatorResetFunc)(VerilatorHandle, int);
 typedef void (*VerilatorAddFunc)(VerilatorHandle, int*, int*, int*, int, int);
 typedef int (*VerilatorReadFunc)(VerilatorHandle, int, int);
@@ -88,10 +89,12 @@ class VerilatorRuntime : public JSONRuntimeBase {
                    const Array<String> const_names)
       : JSONRuntimeBase(symbol_name, graph_json, const_names) {}
 
+  ~VerilatorRuntime();
+
   const char* type_key() const { return "verilator"; }
 
-  /*! \brief load verilator library */
-  void LoadLibrary(const std::string& lib_name);
+  /*! \brief set verilator library */
+  void SetLibrary(const std::string& lib_name);
 
   /*! \brief set the number of reset cycles */
   void SetResetCycles(const int cycles);
@@ -109,6 +112,8 @@ class VerilatorRuntime : public JSONRuntimeBase {
   void Run() override;
 
  private:
+  /*! \brief the verilator library path */
+  String lib_path_;
   /*! \brief the verilator device */
   VerilatorHandle device_{nullptr};
   /*! \brief the verilator library */