DO NOT MERGE diverged version to work with Datomic Cloud aka OpenCANDEL

OPENCandel.md

@@ -0,0 +1,31 @@
+This branch is hacked to work on Datomic Cloud. 
+
+Eventually, it should be merged into the main branch.
+
+# Configuration
+
+Built into the code for now, see https://github.com/Candelbio/pret/blob/cloudy/src/org/candelbio/pret/db.clj#L40
+
+# Credentials
+
+Uses AWS credentials, which must have appropriate permissions. Uses the default profile, but you can change that by editing the configuration.
+
+# Usage
+
+<wd> is a local working directory
+<config> is a standard Pret config file
+<db> is a database name
+
+```
+./pret-dev request-db --import-config <config> --database <db>
+./pret-dev prepare --import-config <config> --working-directory <wd>
+./pret-dev transact --import-config <config> --working-directory <wd> --database <db>
+```
+
+
+# Administration
+
+The `list-dbs` and `delete-db` command should work.
+
+There are some other curation functions in db.clj, not yet hooked to cli. db/print-db-stats is particularly useful.
+

deps.edn

@@ -2,8 +2,10 @@
         com.cognitect/anomalies {:mvn/version "0.1.12"}
         com.cognitect.aws/endpoints {:mvn/version "1.1.11.969"}
         com.cognitect.aws/sns       {:mvn/version "697.2.391.0"}
-        com.datomic/datomic-pro {:mvn/version "1.0.6316"
-                                 :exclusions [org.slf4j/jul-to-slf4j org.slf4j/slf4j-nop]}
+        #_ com.datomic/datomic-pro #_ {:mvn/version "1.0.6316"
+                                       :exclusions [org.slf4j/jul-to-slf4j org.slf4j/slf4j-nop]}
+        com.datomic/client-cloud {:mvn/version "1.0.122"}
+
         com.cognitect.aws/api       {:mvn/version "0.8.505"}
         com.cognitect.aws/s3        {:mvn/version "811.2.858.0"}
         org.clojure/data.csv {:mvn/version "1.0.0"}

src/org/candelbio/pret/cli.clj

@@ -42,7 +42,7 @@
         "                    Requires --database arg."
         "  prepare           Uses an import config file to generate all data needed to run an import."
         "                    Requires --import-config and --working-directory args."
-        "  diff              Generates all changes required to update an existing dataset to match the target."
+        #_ "  diff              Generates all changes required to update an existing dataset to match the target."
         "                    Requires --working-directory and --database arguments."
         "  transact          Transacts all data (as from prepare) for an import job into Datomic."
         "                    Requires --working-directory and --database arguments."
@@ -95,11 +95,11 @@
           (do
             (println "Error requesting database" database)
             (pprint result))
-          (let [db-info (db/fetch-info database)
-                uri (:uri db-info)]
-            (db/init uri)
-            (println "Request successful, created database" (:db-name result))))
+          (do 
+            (db/init database)
+            (println "Request successful, created database" database)))
         :success)
+      #_
       (catch Exception e
         (exit 1 (str "Error encountered creating candel database " database "\n"
                      (when-let [err-data (ex-data e)]
@@ -153,7 +153,7 @@
   [{:keys [target-dir resume database datomic-uri skip-annotations update] :as ctx}]
   (when-not (and datomic-uri database)
     (exit 1 "ERROR: Transact needs a database to transact to."))
-  (print-db-version datomic-uri)
+  #_ (print-db-version datomic-uri)
   (when resume
     (println (str "WARN: Resuming transaction job. This will skip transacting the import job entity. "
                   "Transactions may take awhile to restart as previously successful IDs are found.")))
@@ -168,6 +168,7 @@
                " transactions, entire import job at " target-dir))))
 
 
+#_
 (defn diff
   [{:keys [target-dir resume datomic-uri skip-annotations database] :as ctx}]
   (when-not (and datomic-uri database)
@@ -227,7 +228,7 @@
   {
    "request-db" request-db
    "prepare" prepare
-   "diff" diff
+   ;; "diff" diff
    "transact" transact
    "validate" validate
    "crosscheck-reference" crosscheck-reference
@@ -280,7 +281,7 @@
   [task database-name]
   (when-not (= task "request-db")
     (let [datomic-info (db/fetch-info database-name)
-          datomic-uri (:uri datomic-info)]
+          datomic-uri database-name #_ (:uri datomic-info)]
       (when-not datomic-uri
         (throw (ex-info (str "No such database: " database-name
                              "\nEither name is wrong, or database has been deleted due to inactivity.")
@@ -313,7 +314,7 @@
 
 
 (defn -main [& args]
-  (try
+  (do ; try
     (println "pret version:" (release/version))
     (let [argmap (validate-args args)
           {:keys [exit-message ok?]} argmap]
@@ -327,6 +328,7 @@
           (if (:errors task-results)
             (exit 1 (str "Task: " task " failed "))
             (exit 0 (str "Task: " task " completed."))))))
+    #_
     (catch Throwable t
       (cli.error-handling/report-and-exit t))))
 
@@ -344,4 +346,4 @@
 
   (def val-args (validate-args '("delete-db" "--database" "mt-test3")))
   (def parsed-args (parse-task-args val-args))
-  (delete-db (merge parsed-args (:options parsed-args))))
+  (delete-db (merge parsed-args (:options parsed-args))))

src/org/candelbio/pret/cli/error_handling.clj

@@ -4,11 +4,15 @@
   (:import (java.util.concurrent ExecutionException)))
 
 
+#_
 (defn exit [code msg]
   (println msg)
   (shutdown-agents)
   (System/exit code))
 
+(defn exit [code msg]
+  (println :exiting-not code msg))
+
 (defn report-errors
   "For each key and value in ex-info, report error state."
   [err-map]

src/org/candelbio/pret/db.clj

@@ -1,5 +1,5 @@
 (ns org.candelbio.pret.db
-  (:require [datomic.api :as d]
+  (:require [datomic.client.api :as d]
             [clojure.core.async :as a]
             [org.candelbio.pret.db.backend :as backend]
             [org.candelbio.pret.db.query :as dq]
@@ -27,16 +27,24 @@
 ;;
 
 
+;;; Only used by diff and validation?
+
+#_
 (defn get-connection [info]
-  (d/connect (:uri info)))
+  (backend/connect (:uri info)))
 
 (defn latest-db [info]
-  (d/db (get-connection info)))
+  (backend/db info))
+
+
+
+
+
 
 
 (defn exists? [datomic-uri]
   (try
-    (d/connect datomic-uri)
+    (backend/connect datomic-uri)
     true
     (catch RuntimeException e
       ;; since no ex-info on distributed Datomic, and only get RuntimeException,
@@ -70,27 +78,35 @@
      expected))
 
 
+(defn connect
+  [uri]
+  (backend/connect uri))
+
 (defn apply-schema [datomic-uri]
-  (let [conn (d/connect datomic-uri)
+  (let [conn (connect datomic-uri)
         schema-work schema/schema-txes]
     (doseq [raw-tx schema-work]
       ;; if a schema attr is not indexed, we add index true. this allows us to keep
       ;; schema edn in resources datomic impl agnostic while optimizing on-prem queries.
-      (let [tx (update-in raw-tx [:tx-data]
+      (let [tx raw-tx #_ (update-in raw-tx [:tx-data]
                           (fn [tx-data]
                             (mapv (fn [schema-ent]
                                     (if (and (:db/valueType schema-ent)
                                              (not (:db/unique schema-ent)))
                                        (assoc schema-ent :db/index true)
                                        schema-ent))
                                  tx-data)))]
-        (if (tx-effect? conn tx)
+        (if true #_ (tx-effect? conn tx)
           (do (log/info ::schema (:name tx) " not in database, transacting.")
               (db.tx/sync+retry conn (:tx-data tx)))
           (log/info ::schema "Skipping schema install for: " (:name tx)))))))
 
+
+
+
+
 (defn version [datomic-uri]
-  (let [conn (d/connect datomic-uri)
+  (let [conn (connect datomic-uri)
         db (d/db conn)]
     (-> db
         (d/pull '[:candel.schema/version] :candel/schema)
@@ -99,11 +115,10 @@
 (defn init
   "Loads all base schema, enums, and metamodel into database if necessary."
   [datomic-uri & {:keys [skip-bootstrap seed-data-dir include-proprietary]}]
-  (let [_ (d/create-database datomic-uri)
-        _ (do
-            (log/info "Database created."))
+  (let [;; _ (backend/create-database datomic-uri) ;?? shoudln't this be done earlier
+        ;; _ (do (log/info "Database created."))
         ;; db isn't ready yet if it hasn't been created, this timeout seems sufficient
-        conn (d/connect datomic-uri)
+        conn (connect datomic-uri)
         _ (log/info "Connected to database")]
     (apply-schema datomic-uri)
     (when-not skip-bootstrap
@@ -178,6 +193,12 @@
                    :where
                    [?e :import/txn-id ?id]] db txn-id))))
 
+(defn touch
+  [db id]
+  #_ (d/touch (d/entity db id))
+  nil)    ;TODO
+
+
 (defn head
   "Returns metadata about the last transaction:
 
@@ -191,10 +212,10 @@
                   :where
                   [?tx :db/txInstant]]
                 db))
-        txn-data (d/touch (d/entity db (first txn)))
+        txn-data (touch (first txn))
         import-name (if (contains? txn-data :import/import)
-                      (-> (d/touch (d/entity db (-> (:import/import txn-data)
-                                                    :db/id)))
+                      (-> (touch db (-> (:import/import txn-data)
+                                                    :db/id))
                           :import/name)
                       (throw (ex-info "No datasets transacted"
                                       {:error :no-imports-on-database})))]
@@ -215,7 +236,7 @@
             db)
        (map
          (fn [[name tx-id]]
-           (let [tx (d/touch (d/entity db tx-id))]
+           (let [tx (touch db tx-id)]
              {:timestamp (:db/txInstant tx)
               :import-name name
               :ent-id tx-id})))

src/org/candelbio/pret/db/backend.clj

@@ -1,27 +1,57 @@
 (ns org.candelbio.pret.db.backend
-  (:require [datomic.api :as d]
+  (:require [datomic.client.api :as d]
             [org.candelbio.pret.db.config :as db.config]))
 
 
+;;; MT note: I don't understand why this was carved out of db
+
+;;; Datomic Cloud config
+;;; TODO shouldn't be in code
+(def cfg {:server-type :ion
+          :region "us-east-1" 
+          :system "PublicCANDEL5"
+          ;; This is from the Datomic Cloud ClientApiGatewayEnpoint output
+          :endpoint "https://nazpex6ueb.execute-api.us-east-1.amazonaws.com"
+          ;; :creds-profile "<your_aws_profile_if_not_using_the_default>"
+          })
+
+(def client (d/client cfg))
+
+(defn connect
+  [db-name]
+  #_ (d/connect datomic-uri)
+  (d/connect client {:db-name db-name})
+  )
+
+(defn db
+  [name]
+  (d/db (connect name)))
+
+
+#_
 (defn ddb-base-uri []
   (str "datomic:ddb://"
        (db.config/aws-region) "/"
        (db.config/ddb-table) "/"))
 
+#_
 (defn db-base-uri []
   (or (db.config/base-uri)
       (ddb-base-uri)))
 
 (defn request-db
   [database]
-  (let [uri (str (db-base-uri) database)
-        result (d/create-database uri)]
+  (let [;; uri (str (db-base-uri) database)
+        result #_ (d/create-database uri)
+        (d/create-database client {:db-name database})
+        ]
     (if result
       {:db-name database
        :database database
-       :uri uri}
+       :uri database}                   ;Not really
       {:error "Database already exists!"})))
 
+#_
 (defn delete-db
   [database]
   (let [uri (str (db-base-uri) database)
@@ -33,18 +63,55 @@
        :uri uri}
       {:error "Database not deleted!"})))
 
+(defn delete-db
+  [db]
+  (d/delete-database client {:db-name db}))
+
+#_
 (defn database-info
   "Retrieves the branch database's datomic uri.
   Returns the uri, {:error ...} or throws an exception if the user doesn't have
   access permissions to the branch database."
   [database]
   {:uri (str (db-base-uri) database)})
 
+(defn db-stats
+  [db]
+  (d/db-stats (d/db (d/connect client {:db-name db}))))
+
+;;; This is not terribly useful, but cli could pprint it.
+(defn database-info
+  [db]
+  (db-stats db))
+
+;;; Curation functions.
+
+(defn all-dbs
+  []
+  (d/list-databases client {}))
+
 (defn list-dbs []
-  (try
-    (d/get-database-names (str (db-base-uri) "*"))
-    (catch Exception e
-      {:error (.getMessage e)})))
+  (all-dbs))
+
+(defn all-db-stats
+  []
+  (let [dbs (all-dbs)]
+    (zipmap dbs (map db-stats dbs))))
+
+(defn print-db-stats
+  []
+  (doseq [db (all-dbs)]
+    (prn :db db)
+    (clojure.pprint/pprint (db-stats db))))
+
 
+
 
+(defn delete-all-dbs
+  []
+  (doseq [db (all-dbs)]
+    (delete-db db)))
 
+(defn create-database
+  [name]
+  (d/create-database client {:db-name name}))