From d7dbf45c6d0750f19585dfb4a15ba0f1c2eb1df0 Mon Sep 17 00:00:00 2001 From: Nick Alexander Date: Fri, 20 Jan 2017 15:21:14 -0800 Subject: [PATCH 01/14] Start installing the SQLite store and bootstrapping the datom store. --- .gitignore | 2 + Cargo.toml | 3 + db/Cargo.toml | 23 ++ db/README.md | 3 + db/src/bootstrap.rs | 210 +++++++++++++++++ db/src/db.rs | 541 ++++++++++++++++++++++++++++++++++++++++++++ db/src/debug.rs | 65 ++++++ db/src/errors.rs | 108 +++++++++ db/src/lib.rs | 42 ++++ db/src/schema.rs | 148 ++++++++++++ db/src/types.rs | 251 ++++++++++++++++++++ db/src/values.rs | 61 +++++ tx/src/entities.rs | 2 +- 13 files changed, 1458 insertions(+), 1 deletion(-) create mode 100644 db/Cargo.toml create mode 100644 db/README.md create mode 100644 db/src/bootstrap.rs create mode 100644 db/src/db.rs create mode 100644 db/src/debug.rs create mode 100644 db/src/errors.rs create mode 100644 db/src/lib.rs create mode 100644 db/src/schema.rs create mode 100644 db/src/types.rs create mode 100644 db/src/values.rs diff --git a/.gitignore b/.gitignore index 0fcfa6330..7e07ff1da 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,8 @@ *~ .s* .*.sw* +*.rs.bak +*.bak .hg/ .hgignore .lein-deps-sum diff --git a/Cargo.toml b/Cargo.toml index ed3d80f10..5d0082b03 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,6 +11,9 @@ rusqlite = "0.8.0" [dependencies.edn] path = "edn" +[dependencies.mentat_db] + path = "db" + [dependencies.mentat_query] path = "query" diff --git a/db/Cargo.toml b/db/Cargo.toml new file mode 100644 index 000000000..802083735 --- /dev/null +++ b/db/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "mentat_db" +version = "0.0.1" + +[dependencies] + +[dependencies.edn] +path = "../edn" + +[dependencies.mentat_tx] +path = "../tx" + +[dependencies.mentat_tx_parser] +path = "../tx-parser" + +[dependencies.error-chain] +version = "0.8.0" + +[dependencies.rusqlite] +version = "0.8.0" + +[dependencies.lazy_static] +version = "0.2.2" diff --git a/db/README.md b/db/README.md new file mode 100644 index 000000000..d12a8661c --- /dev/null +++ b/db/README.md @@ -0,0 +1,3 @@ +This sub-crate implements the SQLite database layer: installing, +managing, and migrating forward the SQL schema underlying the datom +store. diff --git a/db/src/bootstrap.rs b/db/src/bootstrap.rs new file mode 100644 index 000000000..73cbd4e85 --- /dev/null +++ b/db/src/bootstrap.rs @@ -0,0 +1,210 @@ +// Copyright 2016 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#![allow(dead_code)] + +use {to_namespaced_keyword}; +use edn; +use edn::types::Value; +use errors::*; +use mentat_tx::entities::Entity; +use mentat_tx_parser; +use types::{IdentMap, Partition, PartitionMap, Schema}; +use values; + +lazy_static! { + static ref V1_IDENTS: [(&'static str, i64); 35] = { + [(":db/ident", 1), + (":db.part/db", 2), + (":db/txInstant", 3), + (":db.install/partition", 4), + (":db.install/valueType", 5), + (":db.install/attribute", 6), + (":db/valueType", 7), + (":db/cardinality", 8), + (":db/unique", 9), + (":db/isComponent", 10), + (":db/index", 11), + (":db/fulltext", 12), + (":db/noHistory", 13), + (":db/add", 14), + (":db/retract", 15), + (":db.part/user", 16), + (":db.part/tx", 17), + (":db/excise", 18), + (":db.excise/attrs", 19), + (":db.excise/beforeT", 20), + (":db.excise/before", 21), + (":db.alter/attribute", 22), + (":db.type/ref", 23), + (":db.type/keyword", 24), + (":db.type/long", 25), + (":db.type/double", 26), + (":db.type/string", 27), + (":db.type/boolean", 28), + (":db.type/instant", 29), + (":db.type/bytes", 30), + (":db.cardinality/one", 31), + (":db.cardinality/many", 32), + (":db.unique/value", 33), + (":db.unique/identity", 34), + (":db/doc", 35), + ] + }; + + static ref V2_IDENTS: [(&'static str, i64); 2] = { + [(":db.schema/version", 36), + (":db.schema/attribute", 37), + ] + }; + + static ref V1_PARTS: [(&'static str, i64, i64); 3] = { + [(":db.part/db", 0, (1 + V1_IDENTS.len() + V2_IDENTS.len()) as i64), + (":db.part/user", 0x10000, 0x10000), + (":db.part/tx", 0x10000000, 0x10000000), + ] + }; + + static ref V2_PARTS: [(&'static str, i64, i64); 0] = { + [] + }; + + static ref V1_SYMBOLIC_SCHEMA: Value = { + let s = r#" +{:db/ident {:db/valueType :db.type/keyword + :db/cardinality :db.cardinality/one + :db/unique :db.unique/identity} + :db.install/partition {:db/valueType :db.type/ref + :db/cardinality :db.cardinality/many} + :db.install/valueType {:db/valueType :db.type/ref + :db/cardinality :db.cardinality/many} + :db.install/attribute {:db/valueType :db.type/ref + :db/cardinality :db.cardinality/many} + ;; TODO: support user-specified functions in the future. + ;; :db.install/function {:db/valueType :db.type/ref + ;; :db/cardinality :db.cardinality/many} + :db/txInstant {:db/valueType :db.type/long + :db/cardinality :db.cardinality/one + :db/index true} + :db/valueType {:db/valueType :db.type/ref + :db/cardinality :db.cardinality/one} + :db/cardinality {:db/valueType :db.type/ref + :db/cardinality :db.cardinality/one} + :db/doc {:db/valueType :db.type/string + :db/cardinality :db.cardinality/one} + :db/unique {:db/valueType :db.type/ref + :db/cardinality :db.cardinality/one} + :db/isComponent {:db/valueType :db.type/boolean + :db/cardinality :db.cardinality/one} + :db/index {:db/valueType :db.type/boolean + :db/cardinality :db.cardinality/one} + :db/fulltext {:db/valueType :db.type/boolean + :db/cardinality :db.cardinality/one} + :db/noHistory {:db/valueType :db.type/boolean + :db/cardinality :db.cardinality/one}}"#; + edn::parse::value(s) + .map_err(|_| ErrorKind::BadBootstrapDefinition("Unable to parse V1_SYMBOLIC_SCHEMA".into())) + .unwrap() + }; + + static ref V2_SYMBOLIC_SCHEMA: Value = { + let s = r#" +{:db.alter/attribute {:db/valueType :db.type/ref + :db/cardinality :db.cardinality/many} + :db.schema/version {:db/valueType :db.type/long + :db/cardinality :db.cardinality/one} + + ;; unique-value because an attribute can only belong to a single + ;; schema fragment. + :db.schema/attribute {:db/valueType :db.type/ref + :db/unique :db.unique/value + :db/cardinality :db.cardinality/many}}"#; + edn::parse::value(s) + .map_err(|_| ErrorKind::BadBootstrapDefinition("Unable to parse V2_SYMBOLIC_SCHEMA".into())) + .unwrap() + }; +} + +/// Convert (ident, entid) pairs into [:db/add IDENT :db/ident IDENT] `Value` instances. +fn idents_to_assertions(idents: &[(&str, i64)]) -> Vec { + idents + .into_iter() + .map(|&(ident, _)| { + let value = Value::NamespacedKeyword(to_namespaced_keyword(&ident).unwrap()); + Value::Vector(vec![values::DB_ADD.clone(), value.clone(), values::DB_IDENT.clone(), value.clone()]) + }) + .collect() +} + +/// Convert {IDENT {:key :value ...} ...} to [[:db/add IDENT :key :value] ...]. +/// In addition, add [:db.add :db.part/db :db.install/attribute IDENT] installation assertions. +fn symbolic_schema_to_assertions(symbolic_schema: &Value) -> Result> { + // Failure here is a coding error, not a runtime error. + let mut assertions: Vec = vec![]; + match *symbolic_schema { + Value::Map(ref m) => { + for (ident, mp) in m { + assertions.push(Value::Vector(vec![values::DB_ADD.clone(), + values::DB_PART_DB.clone(), + values::DB_INSTALL_ATTRIBUTE.clone(), + ident.clone()])); + match *mp { + Value::Map(ref mpp) => { + for (attr, value) in mpp { + assertions.push(Value::Vector(vec![values::DB_ADD.clone(), + ident.clone(), + attr.clone(), + value.clone()])); + } + }, + _ => bail!(ErrorKind::BadBootstrapDefinition("Expected {:db/ident {:db/attr value ...} ...}".into())) + } + } + }, + _ => bail!(ErrorKind::BadBootstrapDefinition("Expected {...}".into())) + } + Ok(assertions) +} + +pub fn bootstrap_partition_map() -> PartitionMap { + V1_PARTS[..].into_iter() + .chain(V2_PARTS[..].into_iter()) + .map(|&(part, start, index)| (part.to_string(), Partition::new(start, index))) + .collect() +} + +pub fn bootstrap_ident_map() -> IdentMap { + V1_IDENTS[..].into_iter() + .chain(V2_IDENTS[..].into_iter()) + .map(|&(ident, entid)| (ident.to_string(), entid)) + .collect() +} + +pub fn bootstrap_schema() -> Schema { + let bootstrap_assertions: Value = Value::Vector([ + symbolic_schema_to_assertions(&V1_SYMBOLIC_SCHEMA).unwrap(), + symbolic_schema_to_assertions(&V2_SYMBOLIC_SCHEMA).unwrap()].concat()); + Schema::from_ident_map_and_assertions(bootstrap_ident_map(), &bootstrap_assertions) + .unwrap() +} + +pub fn bootstrap_entities() -> Vec { + let bootstrap_assertions: Value = Value::Vector([ + symbolic_schema_to_assertions(&V1_SYMBOLIC_SCHEMA).unwrap(), + symbolic_schema_to_assertions(&V2_SYMBOLIC_SCHEMA).unwrap(), + idents_to_assertions(&V1_IDENTS[..]), + idents_to_assertions(&V2_IDENTS[..]), + ].concat()); + + // Failure here is a coding error (since the inputs are fixed), not a runtime error. + // TODO: represent these bootstrap data errors rather than just panicing. + let bootstrap_entities: Vec = mentat_tx_parser::Tx::parse(&[bootstrap_assertions][..]).unwrap(); + return bootstrap_entities; +} diff --git a/db/src/db.rs b/db/src/db.rs new file mode 100644 index 000000000..f67dfc037 --- /dev/null +++ b/db/src/db.rs @@ -0,0 +1,541 @@ +// Copyright 2016 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#![allow(dead_code)] + +use rusqlite; +use rusqlite::types::{ToSql, ToSqlOutput}; + +use bootstrap; +use edn::types::Value; +use errors::*; +use mentat_tx::entities as entmod; +use mentat_tx::entities::Entity; +use types::*; +use values; +use {to_namespaced_keyword}; + +pub fn new_connection() -> rusqlite::Connection { + return rusqlite::Connection::open_in_memory().unwrap(); +} + +/// Version history: +/// +/// 1: initial schema. +/// 2: added :db.schema/version and /attribute in bootstrap; assigned idents 36 and 37, so we bump +/// the part range here; tie bootstrapping to the SQLite user_version. +pub const CURRENT_VERSION: i32 = 2; + +const TRUE: &'static bool = &true; +const FALSE: &'static bool = &false; + +/// Turn an owned bool into a static reference to a bool. +/// +/// `rusqlite` is designed around references to values; this lets us use computed bools easily. +#[inline(always)] +fn to_bool_ref(x: bool) -> &'static bool { + if x { TRUE } else { FALSE } +} + +// /// A typedef of the result returned by many methods. +// pub type Result = result::Result; + +/// SQL statements to be executed, in order, to create the Mentat SQL schema (version 1). +#[cfg_attr(rustfmt, rustfmt_skip)] +const V1_STATEMENTS: [&'static str; 19] = [ + r#"CREATE TABLE datoms (e INTEGER NOT NULL, a SMALLINT NOT NULL, v BLOB NOT NULL, tx INTEGER NOT NULL, + value_type_tag SMALLINT NOT NULL, + index_avet TINYINT NOT NULL DEFAULT 0, index_vaet TINYINT NOT NULL DEFAULT 0, + index_fulltext TINYINT NOT NULL DEFAULT 0, + unique_value TINYINT NOT NULL DEFAULT 0)"#, + r#"CREATE UNIQUE INDEX idx_datoms_eavt ON datoms (e, a, value_type_tag, v)"#, + r#"CREATE UNIQUE INDEX idx_datoms_aevt ON datoms (a, e, value_type_tag, v)"#, + + // Opt-in index: only if a has :db/index true. + r#"CREATE UNIQUE INDEX idx_datoms_avet ON datoms (a, value_type_tag, v, e) WHERE index_avet IS NOT 0"#, + + // Opt-in index: only if a has :db/valueType :db.type/ref. No need for tag here since all + // indexed elements are refs. + r#"CREATE UNIQUE INDEX idx_datoms_vaet ON datoms (v, a, e) WHERE index_vaet IS NOT 0"#, + + // Opt-in index: only if a has :db/fulltext true; thus, it has :db/valueType :db.type/string, + // which is not :db/valueType :db.type/ref. That is, index_vaet and index_fulltext are mutually + // exclusive. + r#"CREATE INDEX idx_datoms_fulltext ON datoms (value_type_tag, v, a, e) WHERE index_fulltext IS NOT 0"#, + + // TODO: possibly remove this index. :db.unique/{value,identity} should be asserted by the + // transactor in all cases, but the index may speed up some of SQLite's query planning. For now, + // it serves to validate the transactor implementation. Note that tag is needed here to + // differentiate, e.g., keywords and strings. + r#"CREATE UNIQUE INDEX idx_datoms_unique_value ON datoms (a, value_type_tag, v) WHERE unique_value IS NOT 0"#, + + r#"CREATE TABLE transactions (e INTEGER NOT NULL, a SMALLINT NOT NULL, v BLOB NOT NULL, tx INTEGER NOT NULL, added TINYINT NOT NULL DEFAULT 1, value_type_tag SMALLINT NOT NULL)"#, + r#"CREATE INDEX idx_transactions_tx ON transactions (tx, added)"#, + + // Fulltext indexing. + // A fulltext indexed value v is an integer rowid referencing fulltext_values. + + // Optional settings: + // tokenize="porter"#, + // prefix='2,3' + // By default we use Unicode-aware tokenizing (particularly for case folding), but preserve + // diacritics. + r#"CREATE VIRTUAL TABLE fulltext_values + USING FTS4 (text NOT NULL, searchid INT, tokenize=unicode61 "remove_diacritics=0")"#, + + // This combination of view and triggers allows you to transparently + // update-or-insert into FTS. Just INSERT INTO fulltext_values_view (text, searchid). + r#"CREATE VIEW fulltext_values_view AS SELECT * FROM fulltext_values"#, + r#"CREATE TRIGGER replace_fulltext_searchid + INSTEAD OF INSERT ON fulltext_values_view + WHEN EXISTS (SELECT 1 FROM fulltext_values WHERE text = new.text) + BEGIN + UPDATE fulltext_values SET searchid = new.searchid WHERE text = new.text; + END"#, + r#"CREATE TRIGGER insert_fulltext_searchid + INSTEAD OF INSERT ON fulltext_values_view + WHEN NOT EXISTS (SELECT 1 FROM fulltext_values WHERE text = new.text) + BEGIN + INSERT INTO fulltext_values (text, searchid) VALUES (new.text, new.searchid); + END"#, + + // A view transparently interpolating fulltext indexed values into the datom structure. + r#"CREATE VIEW fulltext_datoms AS + SELECT e, a, fulltext_values.text AS v, tx, value_type_tag, index_avet, index_vaet, index_fulltext, unique_value + FROM datoms, fulltext_values + WHERE datoms.index_fulltext IS NOT 0 AND datoms.v = fulltext_values.rowid"#, + + // A view transparently interpolating all entities (fulltext and non-fulltext) into the datom structure. + r#"CREATE VIEW all_datoms AS + SELECT e, a, v, tx, value_type_tag, index_avet, index_vaet, index_fulltext, unique_value + FROM datoms + WHERE index_fulltext IS 0 + UNION ALL + SELECT e, a, v, tx, value_type_tag, index_avet, index_vaet, index_fulltext, unique_value + FROM fulltext_datoms"#, + + // Materialized views of the schema. + r#"CREATE TABLE idents (ident TEXT NOT NULL PRIMARY KEY, entid INTEGER UNIQUE NOT NULL)"#, + r#"CREATE TABLE schema (ident TEXT NOT NULL, attr TEXT NOT NULL, value BLOB NOT NULL, value_type_tag SMALLINT NOT NULL, + FOREIGN KEY (ident) REFERENCES idents (ident))"#, + r#"CREATE INDEX idx_schema_unique ON schema (ident, attr, value, value_type_tag)"#, + r#"CREATE TABLE parts (part TEXT NOT NULL PRIMARY KEY, start INTEGER NOT NULL, idx INTEGER NOT NULL)"#, + ]; + +/// Additional SQL statements to be executed, in order, to create the Mentat SQL schema (version 2). +/// We assume that the `V1_STATEMENTS` have been successfully executed (in order) before these are +/// executed. +#[cfg_attr(rustfmt, rustfmt_skip)] +const V2_STATEMENTS: [&'static str; 0] = []; + +/// Set the SQLite user version. +/// +/// Mentat manages its own SQL schema version using the user version. See the [SQLite +/// documentation](https://www.sqlite.org/pragma.html#pragma_user_version). +fn set_user_version(conn: &rusqlite::Connection, version: i32) -> Result<()> { + conn.execute(&format!("PRAGMA user_version = {}", version), &[]) + .chain_err(|| "Could not set_user_version") + .map(|_| ()) +} + +/// Get the SQLite user version. +/// +/// Mentat manages its own SQL schema version using the user version. See the [SQLite +/// documentation](https://www.sqlite.org/pragma.html#pragma_user_version). +fn get_user_version(conn: &rusqlite::Connection) -> Result { + conn.query_row("PRAGMA user_version", &[], |row| { + row.get(0) + }) + .chain_err(|| "Could not get_user_version") +} + +// TODO: rename "SQL" functions to align with "datoms" functions. +pub fn create_current_version(conn: &mut rusqlite::Connection) -> Result { + let tx = conn.transaction()?; + + for statement in &V1_STATEMENTS { + tx.execute(statement, &[])?; + } + + for statement in &V2_STATEMENTS { + try!(tx.execute(statement, &[])); + } + + let bootstrap_partition_map = bootstrap::bootstrap_partition_map(); + // TODO: think more carefully about allocating new parts and bitmasking part ranges. + // TODO: install these using bootstrap assertions. It's tricky because the part ranges are implicit. + // TODO: one insert, chunk into 999/3 sections, for safety. + for (part, partition) in bootstrap_partition_map.iter() { + // TODO: Convert "keyword" part to SQL using Value conversion. + try!(tx.execute("INSERT INTO parts VALUES (?, ?, ?)", &[part, &partition.start, &partition.index])); + } + + let bootstrap_db = DB::new(bootstrap_partition_map, bootstrap::bootstrap_schema()); + bootstrap_db.transact_internal(&tx, &bootstrap::bootstrap_entities()[..])?; + + try!(set_user_version(&tx, CURRENT_VERSION)); + let user_version = try!(get_user_version(&tx)); + + // TODO: use the drop semantics to do this automagically? + tx.commit()?; + Ok(user_version) +} + +// (def v2-statements v1-statements) + +// (defn create-temp-tx-lookup-statement [table-name] +// // n.b., v0/value_type_tag0 can be NULL, in which case we look up v from datoms; +// // and the datom columns are NULL into the LEFT JOIN fills them in. +// // The table-name is not escaped in any way, in order to allow r#"temp.dotted" names. +// // TODO: update comment about sv. +// [(str r#"CREATE TABLE IF NOT EXISTS r#" table-name +// r#" (e0 INTEGER NOT NULL, a0 SMALLINT NOT NULL, v0 BLOB NOT NULL, tx0 INTEGER NOT NULL, added0 TINYINT NOT NULL, +// value_type_tag0 SMALLINT NOT NULL, +// index_avet0 TINYINT, index_vaet0 TINYINT, +// index_fulltext0 TINYINT, +// unique_value0 TINYINT, +// sv BLOB, +// svalue_type_tag SMALLINT, +// rid INTEGER, +// e INTEGER, a SMALLINT, v BLOB, tx INTEGER, value_type_tag SMALLINT)")]) + +// (defn create-temp-tx-lookup-eavt-statement [idx-name table-name] +// // Note that the consuming code creates and drops the indexes +// // manually, which makes insertion slightly faster. +// // This index prevents overlapping transactions. +// // The idx-name and table-name are not escaped in any way, in order +// // to allow r#"temp.dotted" names. +// // TODO: drop added0? +// [(str r#"CREATE UNIQUE INDEX IF NOT EXISTS r#"#, +// idx-name +// r#" ON r#"#, +// table-name +// r#" (e0, a0, v0, added0, value_type_tag0) WHERE sv IS NOT NULL")]) + +// (defn from-version 0)]} // Or we'd create-current-version instead. +// {:pre [(< from-version current-version)]} // Or we wouldn't need to update-from-version. +// (println r#"Upgrading database from" from-version r#"to" current-version) +// (s/in-transaction! +// db +// #(go-pair +// // We must only be migrating from v1 to v2. +// (let [statement r#"UPDATE parts SET idx = idx + 2 WHERE part = ?"] +// (try +// ( Result { + if current_version < 0 || CURRENT_VERSION <= current_version { + bail!(ErrorKind::BadSQLiteStoreVersion(current_version)) + } + + let tx = try!(conn.transaction()); + // TODO: actually implement upgrade. + try!(set_user_version(&tx, CURRENT_VERSION)); + let user_version = try!(get_user_version(&tx)); + // TODO: use the drop semantics to do this automagically? + try!(tx.commit()); + + Ok(user_version) +} + +pub fn ensure_current_version(conn: &mut rusqlite::Connection) -> Result { + let user_version = get_user_version(&conn)?; + match user_version { + CURRENT_VERSION => Ok(user_version), + 0 => create_current_version(conn), + v => update_from_version(conn, v), + } +} + +/// Given a SQLite `value` and a `value_type_tag`, return the corresponding EDN `Value`. +pub fn to_edn(value: &rusqlite::types::Value, value_type_tag: &i32) -> Result { + ValueType::from_value_type_tag(value_type_tag) + .ok_or(ErrorKind::BadValueTypeTag(*value_type_tag).into()) + .and_then(|value_type| -> Result { + match (value_type, value) { + (ValueType::Ref, &rusqlite::types::Value::Integer(ref x)) => Ok(Value::Integer(*x)), + (ValueType::Boolean, &rusqlite::types::Value::Integer(ref x)) => Ok(Value::Boolean(0 != *x)), + (ValueType::Instant, &rusqlite::types::Value::Integer(_)) => bail!(ErrorKind::NotYetImplemented(":db.type/instant".into())), + // SQLite distinguishes integral from decimal types, allowing long and double to + // share a tag. That means the `value_type` above might be incorrect: we could see + // a `Long` tag and really have `Double` data. We always trust the tag: Mentat + // should be managing its metadata correctly. + (ValueType::Long, &rusqlite::types::Value::Integer(ref x)) => Ok(Value::Integer(*x)), + (ValueType::Long, &rusqlite::types::Value::Real(ref x)) if value_type_tag == &ValueType::Double.value_type_tag() => Ok(Value::Float((*x).into())), + // This is spurious, since `value_type` never returns `Double`, but let's just + // leave it in case we change things. + (ValueType::Double, &rusqlite::types::Value::Real(ref x)) => Ok(Value::Float((*x).into())), + (ValueType::String, &rusqlite::types::Value::Text(ref x)) => Ok(Value::Text(x.clone())), + (ValueType::UUID, &rusqlite::types::Value::Text(_)) => bail!(ErrorKind::NotYetImplemented(":db.type/uuid".into())), + (ValueType::URI, &rusqlite::types::Value::Text(_)) => bail!(ErrorKind::NotYetImplemented(":db.type/uri".into())), + (ValueType::Keyword, &rusqlite::types::Value::Text(ref x)) => Ok(Value::Text(x.clone())), + (_, value) => bail!(ErrorKind::BadValueAndTagPair(value.clone(), *value_type_tag)), + } + }) +} + +/// A `Value` that can be converted `ToSql`. +/// +/// This is just working around Rust's refusal to allow us to implement a trait we don't originate +/// for a type we don't originate. +/// +/// TODO: &Value for efficiency. +struct SqlValueWrapper(Value); + +impl ToSql for SqlValueWrapper { + fn to_sql(&self) -> rusqlite::Result { + match self.0 { + Value::Nil => Ok(ToSqlOutput::from(rusqlite::types::Value::Null)), + Value::Boolean(x) => Ok(ToSqlOutput::from(x)), + Value::Integer(x) => Ok(ToSqlOutput::from(x)), + // TODO: consider using a larger radix to save characters. + Value::BigInteger(ref x) => Ok(ToSqlOutput::from(x.to_str_radix(10))), + Value::Float(ref x) => Ok(ToSqlOutput::from(x.into_inner())), + // TODO: try to avoid this clone. + Value::Text(ref x) => Ok(ToSqlOutput::from(x.clone())), + Value::PlainSymbol(ref x) => Ok(ToSqlOutput::from(x.to_string())), + Value::NamespacedSymbol(ref x) => Ok(ToSqlOutput::from(x.to_string())), + Value::Keyword(ref x) => Ok(ToSqlOutput::from(x.to_string())), + Value::NamespacedKeyword(ref x) => Ok(ToSqlOutput::from(x.to_string())), + Value::Vector(_) => Err(rusqlite::Error::InvalidColumnName(format!("Cannot convert to_sql: {:?}", self.0))), + Value::List(_) => Err(rusqlite::Error::InvalidColumnName(format!("Cannot convert to_sql: {:?}", self.0))), + Value::Set(_) => Err(rusqlite::Error::InvalidColumnName(format!("Cannot convert to_sql: {:?}", self.0))), + Value::Map(_) => Err(rusqlite::Error::InvalidColumnName(format!("Cannot convert to_sql: {:?}", self.0))), + } + } +} + +/// Read the ident map materialized view from the given SQL store. +pub fn read_ident_map(conn: &rusqlite::Connection) -> Result { + let mut stmt: rusqlite::Statement = conn.prepare("SELECT ident, entid FROM idents")?; + let m = stmt.query_and_then(&[], |row| -> Result<(String, Entid)> { + Ok((row.get(0), row.get(1))) + })?.collect(); + m +} + + +/// Read the partition map materialized view from the given SQL store. +pub fn read_partition_map(conn: &rusqlite::Connection) -> Result { + let mut stmt: rusqlite::Statement = try!(conn.prepare("SELECT part, start, idx FROM parts")); + let m = stmt.query_and_then(&[], |row| -> Result<(String, Partition)> { + Ok((row.get_checked(0)?, Partition::new(row.get_checked(1)?, row.get_checked(2)?))) + })?.collect(); + m +} + +/// Read the schema materialized view from the given SQL store. +pub fn read_schema(conn: &rusqlite::Connection, ident_map: &IdentMap) -> Result { + // TODO: consider a less expensive way to do this inversion. + let schema_for_idents = Schema::new(ident_map.clone(), SchemaMap::default()); + + let mut stmt: rusqlite::Statement = conn.prepare("SELECT ident, attr, value, value_type_tag FROM schema")?; + let r: Result> = stmt.query_and_then(&[], |row| { + // Each row looks like :db/index|:db/valueType|28|0. Observe that 28|0 represents a + // :db.type/ref to entid 28, which needs to be converted to an ident. + let symbolic_ident: String = row.get_checked(0)?; + let attr: String = row.get_checked(1)?; + let v: rusqlite::types::Value = row.get_checked(2)?; + let value_type_tag = row.get_checked(3)?; + + // We want a symbolic schema, but most of our values are :db.type/ref attributes. Map those + // entids back to idents. This is ad-hoc since we haven't yet a functional DB instance. + let value = match to_edn(&v, &value_type_tag)? { + Value::Integer(entid) if value_type_tag == ValueType::Ref.value_type_tag() => { + schema_for_idents.get_ident(&entid) + .and_then(|x| to_namespaced_keyword(&x)) + .map(Value::NamespacedKeyword) + .ok_or(rusqlite::Error::InvalidColumnName(format!("Could not map :db.type/ref {} to ident!", entid)))? + }, + value => value + }; + + Ok(Value::Vector(vec![ + values::DB_ADD.clone(), + to_namespaced_keyword(&symbolic_ident).map(Value::NamespacedKeyword).ok_or(rusqlite::Error::InvalidColumnName(format!("XX1!")))?, + to_namespaced_keyword(&attr).map(Value::NamespacedKeyword).ok_or(rusqlite::Error::InvalidColumnName(format!("XX2!")))?, + value])) + })?.collect(); + + r.and_then(|values| Schema::from_ident_map_and_assertions(ident_map.clone(), &Value::Vector(values))) +} + +/// Read the materialized views from the given SQL store and return a Mentat `DB` for querying and +/// applying transactions. +pub fn read_db(conn: &rusqlite::Connection) -> Result { + let partition_map = read_partition_map(conn)?; + let ident_map = read_ident_map(conn)?; + let schema = read_schema(conn, &ident_map)?; + Ok(DB::new(partition_map, schema)) +} + +// pub fn bootstrap(conn: &mut rusqlite::Connection, from_version: i32) -> Result<()> { +// match from_version { +// 0 => { +// try!(conn.execute(&format!("INSERT INTO parts VALUES = {}", "()"), &[][..])); +// Ok(()) +// }, +// // 1 => { +// // }, +// // TODO: find a better error type for this. +// _ => Err(rusqlite::Error::InvalidColumnName(format!("Cannot handle bootstrapping from version {}!", from_version))) +// } +// } + +impl DB { + // TODO: move this to the transactor layer. + pub fn transact_internal(&self, conn: &rusqlite::Connection, entities: &[Entity]) -> Result<()>{ + // TODO: manage :db/tx, write :db/txInstant. + let tx = 1; + let r: Vec> = entities.into_iter().map(|entity: &Entity| -> Result<()> { + match *entity { + Entity::Add { + e: entmod::EntidOrLookupRef::Entid(entmod::Entid::Ident(ref e_)), + a: entmod::Entid::Ident(ref a_), + v: entmod::ValueOrLookupRef::Value(ref v_), + tx: _ } => { + + let mut stmt: rusqlite::Statement = try!(conn.prepare("INSERT INTO datoms(e, a, v, tx, value_type_tag, index_avet, index_vaet, index_fulltext, unique_value) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)")); + let e: i64 = *self.schema.get_entid(&e_.to_string()).ok_or(rusqlite::Error::InvalidColumnName(format!("Could not find entid for ident: {:?}", e_)))?; + let a: i64 = *self.schema.get_entid(&a_.to_string()).ok_or(rusqlite::Error::InvalidColumnName(format!("Could not find entid for ident: {:?}", a_)))?; + let attributes: &Attribute = self.schema.schema_map.get(&a).ok_or(rusqlite::Error::InvalidColumnName(format!("Could not find attributes for entid: {:?}", a)))?; + + let value_type_tag = attributes.value_type.value_type_tag(); + + // We can have [:db/ident :db/ref-attr :db/kw] and need to convert the ident + // :db/kw to an entid. So we need some initial type-checking. + // TODO: do the type-checking and value-mapping comprehensively. + let mut v__ = v_.clone(); + if attributes.value_type == ValueType::Ref { + match *v_ { + Value::Integer(_) => (), + Value::NamespacedKeyword(ref s) => { v__ = self.schema.get_entid(&s.to_string()).map(|&x| Value::Integer(x)).ok_or(rusqlite::Error::InvalidColumnName(format!("Could not find entid for ident: {:?}", e_)))? }, + _ => panic!("bad type"), + } + } + + // TODO: avoid spurious clone. + let v = SqlValueWrapper(v__.clone()); + + // Fun times, type signatures. + let values: [&ToSql; 9] = [&e, &a, &v, &tx, &value_type_tag, &attributes.index, to_bool_ref(attributes.value_type == ValueType::Ref), &attributes.fulltext, &attributes.unique_value]; + stmt.insert(&values[..])?; + Ok(()) + }, + // TODO: find a better error type for this. + _ => panic!(format!("Transacting entity not yet supported: {:?}", entity)) // rusqlite::Error::InvalidColumnName(format!("Not yet implented: entities of form ..."))) + } + }).collect(); + + let x: Result> = r.into_iter().collect(); + x.map(|_| ()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use bootstrap; + use debug; + use types::*; + + // #[test] + // fn test_open_current_version() { + // let mut conn = rusqlite::Connection::open("file:///Users/nalexander/Mozilla/mentat/fixtures/v2empty.db").unwrap(); + // // assert_eq!(ensure_current_version(&mut conn).unwrap(), CURRENT_VERSION); + + // // let mut map = IdentMap::new(); + // // assert_eq!(map.insert("a".into(), 1), None); + + // let partition_map = read_partition_map(&conn).unwrap(); + // // assert_eq!(partition_map, bootstrap_partition_map()); + + // let ident_map = read_ident_map(&conn).unwrap(); + // assert_eq!(ident_map, bootstrap_ident_map()); + + // let schema = read_schema(&conn, &ident_map).unwrap(); + // assert_eq!(schema, Schema::default()); + + // let db = DB { + // partition_map: partition_map, + // schema: schema, + // }; + + // // assert_eq!(ident_map, IdentMap::new()); + // // assert_eq!(read_partition_map(&conn).unwrap(), PartitionMap::new()); + // // assert_eq!(read_schema(&conn, &ident_map).unwrap(), Schema::default()); + + // // TODO: fewer magic numbers! + // assert_eq!(datoms_after(&conn, &db, &0x10000000).unwrap(), vec![]); + // } + + #[test] + fn test_create_current_version() { + // // assert_eq!(bootstrap_schema().unwrap(), Schema::default()); + + // // Ignore result. + // use std::fs; + // let _ = fs::remove_file("/Users/nalexander/Mozilla/mentat/test.db"); + // let mut conn = rusqlite::Connection::open("file:///Users/nalexander/Mozilla/mentat/test.db").unwrap(); + + let mut conn = new_connection(); + + assert_eq!(ensure_current_version(&mut conn).unwrap(), CURRENT_VERSION); + + let bootstrap_db = DB::new(bootstrap::bootstrap_partition_map(), bootstrap::bootstrap_schema()); + // TODO: write materialized view of bootstrapped schema to SQL store. + // let db = read_db(&conn).unwrap(); + // assert_eq!(db, bootstrap_db); + + let datoms = debug::datoms_after(&conn, &bootstrap_db, &0).unwrap(); + assert_eq!(datoms.len(), 88); + } +} diff --git a/db/src/debug.rs b/db/src/debug.rs new file mode 100644 index 000000000..6c583bad0 --- /dev/null +++ b/db/src/debug.rs @@ -0,0 +1,65 @@ +// Copyright 2016 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#![allow(dead_code)] + +/// Low-level functions for testing. + +use rusqlite; + +use {to_namespaced_keyword}; +use edn::types::{Value}; +use mentat_tx::entities::{Entid}; +use types::{DB}; +use db::{to_edn}; +use errors::Result; + +/// Represents an assertion (*datom*) in the store. +#[derive(Clone,Debug,Eq,Hash,Ord,PartialOrd,PartialEq)] +pub struct Datom { + // TODO: generalize this. + e: Entid, + a: Entid, + v: Value, + tx: Option, +} + +/// Return the complete set of datoms in the store, ordered by (e, a, v). +pub fn datoms(conn: &rusqlite::Connection, db: &DB) -> Result> { + // TODO: fewer magic numbers! + datoms_after(conn, db, &0x10000000) +} + +/// Return the set of datoms in the store with transaction ID strictly +/// greater than the given `tx`, ordered by (tx, e, a, v). +pub fn datoms_after(conn: &rusqlite::Connection, db: &DB, tx: &i32) -> Result> { + let mut stmt: rusqlite::Statement = try!(conn.prepare("SELECT e, a, v, value_type_tag FROM datoms WHERE tx > ? ORDER BY tx, e, a, v")); + + // Convert numeric entid to entity Entid. + let to_entid = |x| { + db.schema.get_ident(&x).and_then(|y| to_namespaced_keyword(&y)).map(Entid::Ident).unwrap_or(Entid::Entid(x)) + }; + + let datoms = stmt.query_and_then(&[tx], |row| { + let e: i64 = row.get_checked(0)?; + let a: i64 = row.get_checked(1)?; + let v: rusqlite::types::Value = row.get_checked(2)?; + let value_type_tag: i32 = row.get_checked(3)?; + let value: Value = to_edn(&v, &value_type_tag)?; + + Ok(Datom { + e: to_entid(e), + a: to_entid(a), + v: value, + tx: None, + }) + })?.collect(); + datoms +} diff --git a/db/src/errors.rs b/db/src/errors.rs new file mode 100644 index 000000000..f9bf4a67d --- /dev/null +++ b/db/src/errors.rs @@ -0,0 +1,108 @@ +// Copyright 2016 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#![allow(dead_code)] + +use rusqlite; + +error_chain! { + // The type defined for this error. These are the conventional + // and recommended names, but they can be arbitrarily chosen. + // + // It is also possible to leave this section out entirely, or + // leave it empty, and these names will be used automatically. + types { + Error, ErrorKind, ResultExt, Result; + } + + // Without the `Result` wrapper: + // + // types { + // Error, ErrorKind, ResultExt; + // } + + // // Automatic conversions between this error chain and other + // // error chains. In this case, it will e.g. generate an + // // `ErrorKind` variant called `Another` which in turn contains + // // the `other_error::ErrorKind`, with conversions from + // // `other_error::Error`. + // // + // // Optionally, some attributes can be added to a variant. + // // + // // This section can be empty. + // links { + // Another(other_error::Error, other_error::ErrorKind) #[cfg(unix)]; + // } + + // Automatic conversions between this error chain and other + // error types not defined by the `error_chain!`. These will be + // wrapped in a new error with, in the first case, the + // `ErrorKind::Fmt` variant. The description and cause will + // forward to the description and cause of the original error. + // + // Optionally, some attributes can be added to a variant. + // + // This section can be empty. + foreign_links { + // Fmt(::std::fmt::Error); + // Io(::std::io::Error) #[cfg(unix)]; + Rusqlite(rusqlite::Error); + } + + // Define additional `ErrorKind` variants. The syntax here is + // the same as `quick_error!`, but the `from()` and `cause()` + // syntax is not supported. + errors { + /// Something went wrong at the SQLite level. + RusqliteX { // (t: String) { + description("SQLite error") + // display("SQLite error: '{}'", t) + } + + /// We're just not done yet. Message that the feature is recognized but not yet + /// implemented. + NotYetImplemented(t: String) { + description("not yet implemented") + display("not yet implemented: {}", t) + } + + /// We've got corrupt data in the SQL store: a value_type_tag isn't recognized! + BadValueTypeTag(value_type_tag: i32) { + description("bad value_type_tag") + display("bad value_type_tag: {}", value_type_tag) + } + + /// We've got corrupt data in the SQL store: a value and value_type_tag don't line up. + BadValueAndTagPair(value: rusqlite::types::Value, value_type_tag: i32) { + description("bad (value, value_type_tag) pair") + display("bad (value_type_tag, value) pair: ({}, {:?})", value_type_tag, value.data_type()) + } + + /// The SQLite store user_version isn't recognized. This could be an old version of Mentat + /// trying to open a newer version SQLite store; or it could be a corrupt file; or ... + BadSQLiteStoreVersion(version: i32) { + description("bad SQL store user_version") + display("bad SQL store user_version: {}", version) + } + + /// A bootstrap definition couldn't be parsed or installed. This is a programmer error, not + /// a runtime error. + BadBootstrapDefinition(t: String) { + description("bad bootstrap definition") + display("bad bootstrap definition: '{}'", t) + } + + /// A schema assertion couldn't be parsed. + BadSchemaAssertion(t: String) { + description("bad schema assertion") + display("bad schema assertion: '{}'", t) + } + } +} diff --git a/db/src/lib.rs b/db/src/lib.rs new file mode 100644 index 000000000..bc5498b5a --- /dev/null +++ b/db/src/lib.rs @@ -0,0 +1,42 @@ +// Copyright 2016 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#[macro_use] +extern crate error_chain; +#[macro_use] +extern crate lazy_static; +extern crate rusqlite; + +extern crate edn; +extern crate mentat_tx; +extern crate mentat_tx_parser; + +pub use errors::*; +pub use schema::*; +pub use types::*; + +pub mod db; +mod bootstrap; +mod debug; +mod errors; +mod schema; +mod types; +mod values; + +use edn::symbols; + +pub fn to_namespaced_keyword(s: &str) -> Option { + let splits = [':', '/']; + let mut i = s.split(&splits[..]); + match (i.next(), i.next(), i.next(), i.next()) { + (Some(""), Some(namespace), Some(name), None) => Some(symbols::NamespacedKeyword::new(namespace, name)), + _ => None + } +} diff --git a/db/src/schema.rs b/db/src/schema.rs new file mode 100644 index 000000000..5a8660916 --- /dev/null +++ b/db/src/schema.rs @@ -0,0 +1,148 @@ +// Copyright 2016 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#![allow(dead_code)] + +use edn::types::Value; +use errors::*; +use types::{Attribute, Entid, IdentMap, Schema, SchemaMap, ValueType}; +use values; + +impl Schema { + pub fn get_ident(&self, x: &Entid) -> Option<&String> { + self.entid_map.get(x) + } + + pub fn get_entid(&self, x: &String) -> Option<&Entid> { + self.ident_map.get(x) + } + + pub fn attribute_for_entid(&self, x: &Entid) -> Option<&Attribute> { + self.schema_map.get(x) + } + + /// Turn Value([[IDENT ATTR VALUE] ...]) into a Mentat `Schema`. + pub fn from_ident_map_and_assertions(ident_map: IdentMap, assertions: &Value) -> Result { + // Convert Value([[IDENT ATTR VALUE] ...]) to vec![(IDENT.to_string(), ATTR.to_string(), VALUE), ...]. + let triples: Vec<(String, String, &Value)> = match *assertions { + Value::Vector(ref datoms) => { + datoms.into_iter().map(|datom| { + match datom { + &Value::Vector(ref values) => { + let mut i = values.iter(); + match (i.next(), i.next(), i.next(), i.next(), i.next()) { + (Some(add), Some(&Value::NamespacedKeyword(ref ident)), Some(&Value::NamespacedKeyword(ref attr)), Some(value), None) if *add == *values::DB_ADD => + Ok((ident.to_string(), attr.to_string(), value)), + _ => Err(ErrorKind::BadSchemaAssertion(format!("Expected [[:db/add IDENT ATTR VALUE] ...], got: {:?}", datom))) + } + }, + _ => Err(ErrorKind::BadSchemaAssertion(format!("Expected [[...] ...], got: {:?}", datom))) + } + }).collect() + }, + _ => Err(ErrorKind::BadSchemaAssertion(format!("Expected [...], got: {:?}", assertions))) + }?; + + let mut schema_map = SchemaMap::new(); + for (ident, attr, value) in triples { + let entid: &i64 = ident_map.get(&ident).ok_or(ErrorKind::BadSchemaAssertion(format!("Could not get ")))?; + let attributes = schema_map.entry(*entid).or_insert(Attribute::default()); + + // Yes, this is pretty bonkers. Suggestions appreciated. + match attr.as_str() { + ":db/valueType" => { + if *value == *values::DB_TYPE_REF { + attributes.value_type = ValueType::Ref; + } else if *value == *values::DB_TYPE_BOOLEAN { + attributes.value_type = ValueType::Boolean; + } else if *value == *values::DB_TYPE_INSTANT { + attributes.value_type = ValueType::Instant; + } else if *value == *values::DB_TYPE_LONG { + attributes.value_type = ValueType::Long; + } else if *value == *values::DB_TYPE_STRING { + attributes.value_type = ValueType::String; + } else if *value == *values::DB_TYPE_UUID { + attributes.value_type = ValueType::UUID; + } else if *value == *values::DB_TYPE_URI { + attributes.value_type = ValueType::URI; + } else if *value == *values::DB_TYPE_KEYWORD { + attributes.value_type = ValueType::Keyword; + } else { + bail!(ErrorKind::BadSchemaAssertion(format!("Expected [... :db/valueType :db.type/*] but got [... :db/valueType {:?}]", value))) + } + }, + ":db/cardinality" => { + if *value == *values::DB_CARDINALITY_MANY { + attributes.multival = true; + } else if *value == *values::DB_CARDINALITY_ONE { + attributes.multival = false; + } else { + bail!(ErrorKind::BadSchemaAssertion(format!("Expected [... :db/cardinality :db.cardinality/many|:db.cardinality/one] but got [... :db/cardinality {:?}]", value))) + } + }, + ":db/unique" => { + // TODO: assert that we're indexing? + if *value == *values::DB_UNIQUE_VALUE { + attributes.unique_value = true; + } else if *value == *values::DB_UNIQUE_IDENTITY { + attributes.unique_value = true; + attributes.unique_identity = true; + } else { + bail!(ErrorKind::BadSchemaAssertion(format!("Expected [... :db/unique :db.unique/value|:db.unique/identity] but got [... :db/unique {:?}]", value))) + } + }, + ":db/index" => { + if *value == Value::Boolean(true) { + attributes.index = true; + } else if *value == Value::Boolean(false) { + attributes.index = false; + } else { + bail!(ErrorKind::BadSchemaAssertion(format!("Expected [... :db/index true|false] but got [... :db/index {:?}]", value))) + } + }, + ":db/fulltext" => { + // TODO: check valueType is :db.type/string. + if *value == Value::Boolean(true) { + attributes.index = true; + attributes.fulltext = true; + } else if *value == Value::Boolean(false) { + attributes.fulltext = false; + } else { + bail!(ErrorKind::BadSchemaAssertion(format!("Expected [... :db/fulltext true|false] but got [... :db/fulltext {:?}]", value))) + } + }, + ":db/isComponent" => { + // TODO: check valueType is :db.type/ref. + if *value == Value::Boolean(true) { + attributes.component = true; + } else if *value == Value::Boolean(false) { + attributes.component = false; + } else { + bail!(ErrorKind::BadSchemaAssertion(format!("Expected [... :db/isComponent true|false] but got [... :db/isComponent {:?}]", value))) + } + }, + ":db/doc" => { + // Nothing for now. + }, + ":db/ident" => { + // Nothing for now. + }, + ":db.install/attribute" => { + // Nothing for now. + }, + _ => { + bail!(ErrorKind::BadSchemaAssertion(format!("Do not recognize attribute '{}' for ident '{}'", attr, ident))) + } + } + }; + + Ok(Schema::new(ident_map.clone(), schema_map)) + } +} diff --git a/db/src/types.rs b/db/src/types.rs new file mode 100644 index 000000000..8df6bce36 --- /dev/null +++ b/db/src/types.rs @@ -0,0 +1,251 @@ +// Copyright 2016 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#![allow(dead_code)] + +use std::collections::{BTreeMap}; + +/// Core types defining a Mentat knowledge base. +/// +/// At its core, Mentat maintains a set of assertions of the form entity-attribute-value (EAV). The +/// assertions conform to a schema where-by the given attribute constrains the associated value/set +/// of associated values. +/// +/// ## Assertions +/// +/// Mentat assertions are represented as rows in the `datoms` SQLite table, and each Mentat row +/// representing an assertion are tagged with a numeric representation of :db/valueType. +/// +/// The tag is used to limit queries, and therefore is placed carefully in the relevant indices to +/// allow searching numeric longs and doubles quickly. The tag is also used to convert SQLite +/// values to the correct Mentat value type on query egress. +/// +/// ## Entities and entids +/// +/// A Mentat entity is represented by a *positive* integer. (This agrees with Datomic.) We call +/// such a positive integer an *entid*. +/// +/// ## Partitions +/// +/// Datomic partitions the entid space in order to separate core knowledge base entities required +/// for the healthy function of the system from user-defined entities. Datomic also partitions in +/// order to ensure that certain index walks of related entities are efficient. Mentat follows +/// suit, partitioning into the following partitions: +/// * `:db.part/db`, for core knowledge base entities; +/// * `:db.part/user`, for user-defined entities; +/// * `:db.part/tx`, for transaction entities. +/// You almost certainly want to add new entities in the `:db.part/user` partition. +/// +/// The entid sequence in a given partition is monotonically increasing, although not necessarily +/// contiguous. That is, it is possible for a specific entid to have never been present in the +/// system, even though its predecessor and successor is present. + +// #[derive(Debug)] +// pub enum Error { +// RusqliteError(rusqlite::Error), +// BadSchemaAssertion(String), +// BadBootstrapDefinition(String), +// } + +// impl From for Error { +// fn from(e: rusqlite::Error) -> Error { +// Error::RusqliteError(e) +// } +// } + +/// Represents one entid in the entid space. +/// +/// Per https://www.sqlite.org/datatype3.html (see also http://stackoverflow.com/a/8499544), SQLite +/// stores signed integers up to 64 bits in size. Since u32 is not appropriate for our use case, we +/// use i64 rather than manually truncating u64 to u63 and casting to i64 throughout the codebase. +pub type Entid = i64; + +/// The attribute of each Mentat assertion has an :db/valueType constraining the value to a +/// particular domain. Mentat recognizes the following :db/valueType values. +#[derive(Clone,Debug,Eq,Hash,Ord,PartialOrd,PartialEq)] +pub enum ValueType { + Ref, + Boolean, + Instant, + Long, + Double, + String, + UUID, + URI, + Keyword, +} + +impl ValueType { + pub fn value_type_tag(&self) -> i32 { + match *self { + ValueType::Ref => 0, + ValueType::Boolean => 1, + ValueType::Instant => 4, + ValueType::Long => 5, // SQLite distinguishes integral from decimal types, allowing long and double to share a tag. + ValueType::Double => 5, // SQLite distinguishes integral from decimal types, allowing long and double to share a tag. + ValueType::String => 10, + ValueType::UUID => 11, + ValueType::URI => 12, + ValueType::Keyword => 13, + } + } + + pub fn from_value_type_tag(value_type_tag: &i32) -> Option { + match *value_type_tag { + 0 => Some(ValueType::Ref), + 1 => Some(ValueType::Boolean), + 4 => Some(ValueType::Instant), + 5 => Some(ValueType::Long), // SQLite distinguishes integral from decimal types, allowing long and double to share a tag. + // 5 => Some(ValueType::Double), + 10 => Some(ValueType::String), + 11 => Some(ValueType::UUID), + 12 => Some(ValueType::URI), + 13 => Some(ValueType::Keyword), + _ => None + } + } +} + +/// Represents one partition of the entid space. +#[derive(Clone,Debug,Eq,Hash,Ord,PartialOrd,PartialEq)] +pub struct Partition { + /// The first entid in the partition. + pub start: i64, + /// The next entid to be allocated in the partition. + pub index: i64, +} + +impl Partition { + pub fn new(start: i64, next: i64) -> Partition { + assert!(start <= next, "A partition represents a monotonic increasing sequence of entids."); + Partition { start: start, index: next } + } +} + +/// Map partition names to `Partition` instances. +pub type PartitionMap = BTreeMap; + +/// A Mentat schema attribute has a value type and several other flags determining how assertions +/// with the attribute are interpreted. +/// +/// TODO: consider packing this into a bitfield or similar. +#[derive(Clone,Debug,Eq,Hash,Ord,PartialOrd,PartialEq)] +pub struct Attribute { + /// The associated value type, i.e., `:db/valueType`? + pub value_type: ValueType, + /// `true` if this attribute is multi-valued, i.e., it is `:db/cardinality + /// :db.cardinality/many`. `false` if this attribute is single-valued (the default), i.e., it + /// is `:db/cardinality :db.cardinality/one`. + pub multival: bool, + /// `true` if this attribute is unique-value, i.e., it is `:db/unique :db.unique/value`. + /// + /// *Unique-value* means that there is at most one assertion with the attribute and a + /// particular value in the datom store. + pub unique_value: bool, + /// `true` if this attribute is unique-identity, i.e., it is `:db/unique :db.unique/identity`. + /// + /// Unique-identity attributes always have value type `Ref`. + /// + /// *Unique-identity* means that the attribute is *unique-value* and that they can be used in + /// lookup-refs and will automatically upsert where appropriate. + pub unique_identity: bool, + /// `true` if this attribute is automatically indexed, i.e., it is `:db/indexing true`. + pub index: bool, + /// `true` if this attribute is automatically fulltext indexed, i.e., it is `:db/fulltext true`. + /// + /// Fulltext attributes always have string values. + pub fulltext: bool, + /// `true` if this attribute is a component, i.e., it is `:db/isComponent true`. + /// + /// Component attributes always have value type `Ref`. + /// + /// They are used to compose entities from component sub-entities: they are fetched recursively + /// by pull expressions, and they are automatically recursively deleted where appropriate. + pub component: bool, +} + +impl Default for Attribute { + fn default() -> Attribute { + Attribute { + // There's no particular reason to favour one value type, so Ref it is. + value_type: ValueType::Ref, + fulltext: false, + index: false, + multival: false, + unique_value: false, + unique_identity: false, + component: false, + } + } +} + +/// Map `String` idents (`:db/ident`) to positive integer entids (`1`). +pub type IdentMap = BTreeMap; + +/// Map positive integer entids (`1`) to `String` idents (`:db/ident`). +pub type EntidMap = BTreeMap; + +/// Map attribute entids to `Attribute` instances. +pub type SchemaMap = BTreeMap; + +/// Represents a Mentat schema. +/// +/// Maintains the mapping between string idents and positive integer entids; and exposes the schema +/// flags associated to a given entid (equivalently, ident). +/// +/// TODO: consider a single bi-directional map instead of separate ident->entid and entid->ident +/// maps. +#[derive(Clone,Debug,Default,Eq,Hash,Ord,PartialOrd,PartialEq)] +pub struct Schema { + /// Map entid->ident. + /// + /// Invariant: is the inverse map of `ident_map`. + pub entid_map: EntidMap, + /// Map ident->entid. + /// + /// Invariant: is the inverse map of `entid_map`. + pub ident_map: IdentMap, + /// Map entid->attribute flags. + /// + /// Invariant: key-set is the same as the key-set of `entid_map` (equivalently, the value-set of + /// `ident_map`). + pub schema_map: SchemaMap, +} + +impl Schema { + pub fn new(ident_map: IdentMap, schema_map: SchemaMap) -> Schema { + let entid_map: EntidMap = ident_map.iter().map(|(k, v)| (v.clone(), k.clone())).collect(); + Schema { + ident_map: ident_map, + entid_map: entid_map, + schema_map: schema_map, + } + } +} + +/// Represents the metadata required to query from, or apply transactions to, a Mentat store. +#[derive(Clone,Debug,Default,Eq,Hash,Ord,PartialOrd,PartialEq)] +pub struct DB { + /// Map partition name->`Partition`. + /// + /// TODO: represent partitions as entids. + pub partition_map: PartitionMap, + /// The schema of the store. + pub schema: Schema, +} + +impl DB { + pub fn new(partition_map: PartitionMap, schema: Schema) -> DB { + DB { + partition_map: partition_map, + schema: schema + } + } +} diff --git a/db/src/values.rs b/db/src/values.rs new file mode 100644 index 000000000..958816a0d --- /dev/null +++ b/db/src/values.rs @@ -0,0 +1,61 @@ +// Copyright 2016 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#![allow(dead_code)] + +/// Literal `Value` instances in the the "db" namespace. +/// +/// Used through-out the transactor to match core DB constructs. + +use edn::types::Value; +use edn::symbols; + +/// Declare a lazy static `ident` of type `Value::NamespacedKeyword` with the given `namespace` and +/// `name`. +/// +/// It may look surprising that we declare a new `lazy_static!` block rather than including +/// invocations inside an existing `lazy_static!` block. The latter cannot be done, since macros +/// are expanded outside-in. Looking at the `lazy_static!` source suggests that there is no harm in +/// repeating that macro, since internally a multi-`static` block is expanded into many +/// single-`static` blocks. +/// +/// TODO: take just ":db.part/db" and define DB_PART_DB using "db.part" and "db". +macro_rules! lazy_static_namespaced_keyword_value ( + ($tag:ident, $namespace:expr, $name:expr) => ( + lazy_static! { + pub static ref $tag: Value = { + Value::NamespacedKeyword(symbols::NamespacedKeyword::new($namespace, $name)) + }; + } + ) +); + +lazy_static_namespaced_keyword_value!(DB_ADD, "db", "add"); +lazy_static_namespaced_keyword_value!(DB_ALTER_ATTRIBUTE, "db.alter", "attribute"); +lazy_static_namespaced_keyword_value!(DB_CARDINALITY, "db", "cardinality"); +lazy_static_namespaced_keyword_value!(DB_CARDINALITY_MANY, "db.cardinality", "many"); +lazy_static_namespaced_keyword_value!(DB_CARDINALITY_ONE, "db.cardinality", "one"); +lazy_static_namespaced_keyword_value!(DB_IDENT, "db", "ident"); +lazy_static_namespaced_keyword_value!(DB_INSTALL_ATTRIBUTE, "db.install", "attribute"); +lazy_static_namespaced_keyword_value!(DB_PART_DB, "db.part", "db"); +lazy_static_namespaced_keyword_value!(DB_RETRACT, "db", "retract"); +lazy_static_namespaced_keyword_value!(DB_TYPE_BOOLEAN, "db.type", "boolean"); +lazy_static_namespaced_keyword_value!(DB_TYPE_DOUBLE, "db.type", "double"); +lazy_static_namespaced_keyword_value!(DB_TYPE_INSTANT, "db.type", "instant"); +lazy_static_namespaced_keyword_value!(DB_TYPE_KEYWORD, "db.type", "keyword"); +lazy_static_namespaced_keyword_value!(DB_TYPE_LONG, "db.type", "long"); +lazy_static_namespaced_keyword_value!(DB_TYPE_REF, "db.type", "ref"); +lazy_static_namespaced_keyword_value!(DB_TYPE_STRING, "db.type", "string"); +lazy_static_namespaced_keyword_value!(DB_TYPE_URI, "db.type", "uri"); +lazy_static_namespaced_keyword_value!(DB_TYPE_UUID, "db.type", "uuid"); +lazy_static_namespaced_keyword_value!(DB_UNIQUE, "db", "unique"); +lazy_static_namespaced_keyword_value!(DB_UNIQUE_IDENTITY, "db.unique", "identity"); +lazy_static_namespaced_keyword_value!(DB_UNIQUE_VALUE, "db.unique", "value"); +lazy_static_namespaced_keyword_value!(DB_VALUE_TYPE, "db", "valueType"); diff --git a/tx/src/entities.rs b/tx/src/entities.rs index 3c99b622c..fd78d5015 100644 --- a/tx/src/entities.rs +++ b/tx/src/entities.rs @@ -15,7 +15,7 @@ extern crate edn; use self::edn::types::Value; use self::edn::symbols::NamespacedKeyword; -#[derive(Clone, Debug, PartialEq)] +#[derive(Clone,Debug,Eq,Hash,Ord,PartialOrd,PartialEq)] pub enum Entid { Entid(i64), Ident(NamespacedKeyword), From b824b84ba1069627e469b9741575b3b3de9ac259 Mon Sep 17 00:00:00 2001 From: Nick Alexander Date: Mon, 23 Jan 2017 13:03:35 -0800 Subject: [PATCH 02/14] Review comment: Pre: Expose Clojure's merge on Value instances. --- edn/src/lib.rs | 1 + edn/src/utils.rs | 31 +++++++++++++++++++++++++++++++ edn/tests/tests.rs | 37 +++++++++++++++++++++++++++++++++++++ 3 files changed, 69 insertions(+) create mode 100644 edn/src/utils.rs diff --git a/edn/src/lib.rs b/edn/src/lib.rs index 4faf6853b..002ea4c27 100644 --- a/edn/src/lib.rs +++ b/edn/src/lib.rs @@ -15,6 +15,7 @@ extern crate num; pub mod symbols; pub mod types; +pub mod utils; pub mod parse { include!(concat!(env!("OUT_DIR"), "/edn.rs")); diff --git a/edn/src/utils.rs b/edn/src/utils.rs new file mode 100644 index 000000000..f5ea960f0 --- /dev/null +++ b/edn/src/utils.rs @@ -0,0 +1,31 @@ +// Copyright 2016 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#![allow(dead_code)] + +use types::Value; + +/// Merge the EDN `Value::Map` instance `right` into `left`. Returns `None` if either `left` or +/// `right` is not a `Value::Map`. +/// +/// Keys present in `right` overwrite keys present in `left`. See also +/// https://clojuredocs.org/clojure.core/merge. +/// +/// TODO: implement `merge` for [Value], following the `concat`/`SliceConcatExt` pattern. +pub fn merge(left: &Value, right: &Value) -> Option { + match (left, right) { + (&Value::Map(ref l), &Value::Map(ref r)) => { + let mut result = l.clone(); + result.extend(r.clone().into_iter()); + Some(Value::Map(result)) + } + _ => None + } +} diff --git a/edn/tests/tests.rs b/edn/tests/tests.rs index ade8c7b18..1707cd2b0 100644 --- a/edn/tests/tests.rs +++ b/edn/tests/tests.rs @@ -21,6 +21,7 @@ use edn::symbols; use edn::types::Value; use edn::types::Value::*; use edn::parse::*; +use edn::utils; // Helper for making wrapped keywords with a namespace. fn k_ns(ns: &str, name: &str) -> Value { @@ -850,6 +851,42 @@ fn test_spurious_commas() { assert_eq!(value("[3,,]"), result); } +#[test] +fn test_utils_merge() { + // Take BTreeMap instances, wrap into Value::Map instances. + let test = |left: &BTreeMap, right: &BTreeMap, expected: &BTreeMap| { + let l = Value::Map(left.clone()); + let r = Value::Map(right.clone()); + let result = utils::merge(&l, &r).unwrap(); + let e = Value::Map(expected.clone()); + assert_eq!(result, e); + }; + + let mut left = BTreeMap::new(); + left.insert(Value::Integer(1), Value::Integer(1)); + left.insert(Value::Text("a".into()), Value::Text("a".into())); + let mut right = BTreeMap::new(); + right.insert(Value::Integer(2), Value::Integer(2)); + right.insert(Value::Text("a".into()), Value::Text("b".into())); + + let mut expected = BTreeMap::new(); + expected.insert(Value::Integer(1), Value::Integer(1)); + expected.insert(Value::Integer(2), Value::Integer(2)); + expected.insert(Value::Text("a".into()), Value::Text("b".into())); + + let mut expected = BTreeMap::new(); + expected.insert(Value::Integer(1), Value::Integer(1)); + expected.insert(Value::Integer(2), Value::Integer(2)); + expected.insert(Value::Text("a".into()), Value::Text("b".into())); + test(&left, &right, &expected); + + let mut expected = BTreeMap::new(); + expected.insert(Value::Integer(1), Value::Integer(1)); + expected.insert(Value::Integer(2), Value::Integer(2)); + expected.insert(Value::Text("a".into()), Value::Text("a".into())); + test(&right, &left, &expected); +} + /* // Handy templates for creating test cases follow: From 4d0c3e6b5df52d76800f6ec299a36618d4069ad6 Mon Sep 17 00:00:00 2001 From: Nick Alexander Date: Mon, 23 Jan 2017 11:49:00 -0800 Subject: [PATCH 03/14] Review comment: Decomplect V2_IDENTS. --- db/src/bootstrap.rs | 85 ++++++++++++++++++++++----------------------- 1 file changed, 42 insertions(+), 43 deletions(-) diff --git a/db/src/bootstrap.rs b/db/src/bootstrap.rs index 73cbd4e85..d33ed0ea1 100644 --- a/db/src/bootstrap.rs +++ b/db/src/bootstrap.rs @@ -20,49 +20,50 @@ use types::{IdentMap, Partition, PartitionMap, Schema}; use values; lazy_static! { - static ref V1_IDENTS: [(&'static str, i64); 35] = { - [(":db/ident", 1), - (":db.part/db", 2), - (":db/txInstant", 3), - (":db.install/partition", 4), - (":db.install/valueType", 5), - (":db.install/attribute", 6), - (":db/valueType", 7), - (":db/cardinality", 8), - (":db/unique", 9), - (":db/isComponent", 10), - (":db/index", 11), - (":db/fulltext", 12), - (":db/noHistory", 13), - (":db/add", 14), - (":db/retract", 15), - (":db.part/user", 16), - (":db.part/tx", 17), - (":db/excise", 18), - (":db.excise/attrs", 19), - (":db.excise/beforeT", 20), - (":db.excise/before", 21), - (":db.alter/attribute", 22), - (":db.type/ref", 23), - (":db.type/keyword", 24), - (":db.type/long", 25), - (":db.type/double", 26), - (":db.type/string", 27), - (":db.type/boolean", 28), - (":db.type/instant", 29), - (":db.type/bytes", 30), - (":db.cardinality/one", 31), - (":db.cardinality/many", 32), - (":db.unique/value", 33), - (":db.unique/identity", 34), - (":db/doc", 35), + static ref V1_IDENTS: Vec<(&'static str, i64)> = { + vec![(":db/ident", 1), + (":db.part/db", 2), + (":db/txInstant", 3), + (":db.install/partition", 4), + (":db.install/valueType", 5), + (":db.install/attribute", 6), + (":db/valueType", 7), + (":db/cardinality", 8), + (":db/unique", 9), + (":db/isComponent", 10), + (":db/index", 11), + (":db/fulltext", 12), + (":db/noHistory", 13), + (":db/add", 14), + (":db/retract", 15), + (":db.part/user", 16), + (":db.part/tx", 17), + (":db/excise", 18), + (":db.excise/attrs", 19), + (":db.excise/beforeT", 20), + (":db.excise/before", 21), + (":db.alter/attribute", 22), + (":db.type/ref", 23), + (":db.type/keyword", 24), + (":db.type/long", 25), + (":db.type/double", 26), + (":db.type/string", 27), + (":db.type/boolean", 28), + (":db.type/instant", 29), + (":db.type/bytes", 30), + (":db.cardinality/one", 31), + (":db.cardinality/many", 32), + (":db.unique/value", 33), + (":db.unique/identity", 34), + (":db/doc", 35), ] }; - static ref V2_IDENTS: [(&'static str, i64); 2] = { - [(":db.schema/version", 36), - (":db.schema/attribute", 37), - ] + static ref V2_IDENTS: Vec<(&'static str, i64)> = { + [(*V1_IDENTS).clone(), + vec![(":db.schema/version", 36), + (":db.schema/attribute", 37), + ]].concat() }; static ref V1_PARTS: [(&'static str, i64, i64); 3] = { @@ -181,8 +182,7 @@ pub fn bootstrap_partition_map() -> PartitionMap { } pub fn bootstrap_ident_map() -> IdentMap { - V1_IDENTS[..].into_iter() - .chain(V2_IDENTS[..].into_iter()) + V2_IDENTS[..].iter() .map(|&(ident, entid)| (ident.to_string(), entid)) .collect() } @@ -199,7 +199,6 @@ pub fn bootstrap_entities() -> Vec { let bootstrap_assertions: Value = Value::Vector([ symbolic_schema_to_assertions(&V1_SYMBOLIC_SCHEMA).unwrap(), symbolic_schema_to_assertions(&V2_SYMBOLIC_SCHEMA).unwrap(), - idents_to_assertions(&V1_IDENTS[..]), idents_to_assertions(&V2_IDENTS[..]), ].concat()); From 524b539e8ede56f6cecee4e76181f80f3d4751a0 Mon Sep 17 00:00:00 2001 From: Nick Alexander Date: Mon, 23 Jan 2017 11:51:13 -0800 Subject: [PATCH 04/14] Review comment: Decomplect V2_PARTS. --- db/src/bootstrap.rs | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/db/src/bootstrap.rs b/db/src/bootstrap.rs index d33ed0ea1..b6daacf24 100644 --- a/db/src/bootstrap.rs +++ b/db/src/bootstrap.rs @@ -66,15 +66,18 @@ lazy_static! { ]].concat() }; - static ref V1_PARTS: [(&'static str, i64, i64); 3] = { - [(":db.part/db", 0, (1 + V1_IDENTS.len() + V2_IDENTS.len()) as i64), - (":db.part/user", 0x10000, 0x10000), - (":db.part/tx", 0x10000000, 0x10000000), - ] + static ref V1_PARTS: Vec<(&'static str, i64, i64)> = { + vec![(":db.part/db", 0, (1 + V1_IDENTS.len()) as i64), + (":db.part/user", 0x10000, 0x10000), + (":db.part/tx", 0x10000000, 0x10000000), + ] }; - static ref V2_PARTS: [(&'static str, i64, i64); 0] = { - [] + static ref V2_PARTS: Vec<(&'static str, i64, i64)> = { + vec![(":db.part/db", 0, (1 + V2_IDENTS.len()) as i64), + (":db.part/user", 0x10000, 0x10000), + (":db.part/tx", 0x10000000, 0x10000000), + ] }; static ref V1_SYMBOLIC_SCHEMA: Value = { @@ -175,8 +178,7 @@ fn symbolic_schema_to_assertions(symbolic_schema: &Value) -> Result> } pub fn bootstrap_partition_map() -> PartitionMap { - V1_PARTS[..].into_iter() - .chain(V2_PARTS[..].into_iter()) + V2_PARTS[..].iter() .map(|&(part, start, index)| (part.to_string(), Partition::new(start, index))) .collect() } From 8a67cd9bce446f1302b13e3ce5e94258a88cac45 Mon Sep 17 00:00:00 2001 From: Nick Alexander Date: Mon, 23 Jan 2017 13:09:39 -0800 Subject: [PATCH 05/14] Review comment: Decomplect V2_SYMBOLIC_SCHEMA. --- db/src/bootstrap.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/db/src/bootstrap.rs b/db/src/bootstrap.rs index b6daacf24..0c0586855 100644 --- a/db/src/bootstrap.rs +++ b/db/src/bootstrap.rs @@ -130,8 +130,11 @@ lazy_static! { :db.schema/attribute {:db/valueType :db.type/ref :db/unique :db.unique/value :db/cardinality :db.cardinality/many}}"#; - edn::parse::value(s) + let right = edn::parse::value(s) .map_err(|_| ErrorKind::BadBootstrapDefinition("Unable to parse V2_SYMBOLIC_SCHEMA".into())) + .unwrap(); + edn::utils::merge(&V1_SYMBOLIC_SCHEMA, &right) + .ok_or(ErrorKind::BadBootstrapDefinition("Unable to parse V2_SYMBOLIC_SCHEMA".into())) .unwrap() }; } @@ -190,16 +193,13 @@ pub fn bootstrap_ident_map() -> IdentMap { } pub fn bootstrap_schema() -> Schema { - let bootstrap_assertions: Value = Value::Vector([ - symbolic_schema_to_assertions(&V1_SYMBOLIC_SCHEMA).unwrap(), - symbolic_schema_to_assertions(&V2_SYMBOLIC_SCHEMA).unwrap()].concat()); + let bootstrap_assertions: Value = Value::Vector(symbolic_schema_to_assertions(&V2_SYMBOLIC_SCHEMA).unwrap()); Schema::from_ident_map_and_assertions(bootstrap_ident_map(), &bootstrap_assertions) .unwrap() } pub fn bootstrap_entities() -> Vec { let bootstrap_assertions: Value = Value::Vector([ - symbolic_schema_to_assertions(&V1_SYMBOLIC_SCHEMA).unwrap(), symbolic_schema_to_assertions(&V2_SYMBOLIC_SCHEMA).unwrap(), idents_to_assertions(&V2_IDENTS[..]), ].concat()); From f1769338e7f3a1d205c6c78f31508eefed3c4a02 Mon Sep 17 00:00:00 2001 From: Nick Alexander Date: Mon, 23 Jan 2017 13:18:11 -0800 Subject: [PATCH 06/14] Review comment: Decomplect V1_STATEMENTS. --- db/src/db.rs | 180 ++++++++++++++++++++++++--------------------------- 1 file changed, 85 insertions(+), 95 deletions(-) diff --git a/db/src/db.rs b/db/src/db.rs index f67dfc037..46394ca80 100644 --- a/db/src/db.rs +++ b/db/src/db.rs @@ -44,96 +44,90 @@ fn to_bool_ref(x: bool) -> &'static bool { if x { TRUE } else { FALSE } } -// /// A typedef of the result returned by many methods. -// pub type Result = result::Result; - -/// SQL statements to be executed, in order, to create the Mentat SQL schema (version 1). -#[cfg_attr(rustfmt, rustfmt_skip)] -const V1_STATEMENTS: [&'static str; 19] = [ - r#"CREATE TABLE datoms (e INTEGER NOT NULL, a SMALLINT NOT NULL, v BLOB NOT NULL, tx INTEGER NOT NULL, - value_type_tag SMALLINT NOT NULL, - index_avet TINYINT NOT NULL DEFAULT 0, index_vaet TINYINT NOT NULL DEFAULT 0, - index_fulltext TINYINT NOT NULL DEFAULT 0, - unique_value TINYINT NOT NULL DEFAULT 0)"#, - r#"CREATE UNIQUE INDEX idx_datoms_eavt ON datoms (e, a, value_type_tag, v)"#, - r#"CREATE UNIQUE INDEX idx_datoms_aevt ON datoms (a, e, value_type_tag, v)"#, - - // Opt-in index: only if a has :db/index true. - r#"CREATE UNIQUE INDEX idx_datoms_avet ON datoms (a, value_type_tag, v, e) WHERE index_avet IS NOT 0"#, - - // Opt-in index: only if a has :db/valueType :db.type/ref. No need for tag here since all - // indexed elements are refs. - r#"CREATE UNIQUE INDEX idx_datoms_vaet ON datoms (v, a, e) WHERE index_vaet IS NOT 0"#, - - // Opt-in index: only if a has :db/fulltext true; thus, it has :db/valueType :db.type/string, - // which is not :db/valueType :db.type/ref. That is, index_vaet and index_fulltext are mutually - // exclusive. - r#"CREATE INDEX idx_datoms_fulltext ON datoms (value_type_tag, v, a, e) WHERE index_fulltext IS NOT 0"#, - - // TODO: possibly remove this index. :db.unique/{value,identity} should be asserted by the - // transactor in all cases, but the index may speed up some of SQLite's query planning. For now, - // it serves to validate the transactor implementation. Note that tag is needed here to - // differentiate, e.g., keywords and strings. - r#"CREATE UNIQUE INDEX idx_datoms_unique_value ON datoms (a, value_type_tag, v) WHERE unique_value IS NOT 0"#, - - r#"CREATE TABLE transactions (e INTEGER NOT NULL, a SMALLINT NOT NULL, v BLOB NOT NULL, tx INTEGER NOT NULL, added TINYINT NOT NULL DEFAULT 1, value_type_tag SMALLINT NOT NULL)"#, - r#"CREATE INDEX idx_transactions_tx ON transactions (tx, added)"#, - - // Fulltext indexing. - // A fulltext indexed value v is an integer rowid referencing fulltext_values. - - // Optional settings: - // tokenize="porter"#, - // prefix='2,3' - // By default we use Unicode-aware tokenizing (particularly for case folding), but preserve - // diacritics. - r#"CREATE VIRTUAL TABLE fulltext_values - USING FTS4 (text NOT NULL, searchid INT, tokenize=unicode61 "remove_diacritics=0")"#, - - // This combination of view and triggers allows you to transparently - // update-or-insert into FTS. Just INSERT INTO fulltext_values_view (text, searchid). - r#"CREATE VIEW fulltext_values_view AS SELECT * FROM fulltext_values"#, - r#"CREATE TRIGGER replace_fulltext_searchid - INSTEAD OF INSERT ON fulltext_values_view - WHEN EXISTS (SELECT 1 FROM fulltext_values WHERE text = new.text) - BEGIN - UPDATE fulltext_values SET searchid = new.searchid WHERE text = new.text; - END"#, - r#"CREATE TRIGGER insert_fulltext_searchid - INSTEAD OF INSERT ON fulltext_values_view - WHEN NOT EXISTS (SELECT 1 FROM fulltext_values WHERE text = new.text) - BEGIN - INSERT INTO fulltext_values (text, searchid) VALUES (new.text, new.searchid); - END"#, - - // A view transparently interpolating fulltext indexed values into the datom structure. - r#"CREATE VIEW fulltext_datoms AS - SELECT e, a, fulltext_values.text AS v, tx, value_type_tag, index_avet, index_vaet, index_fulltext, unique_value - FROM datoms, fulltext_values - WHERE datoms.index_fulltext IS NOT 0 AND datoms.v = fulltext_values.rowid"#, - - // A view transparently interpolating all entities (fulltext and non-fulltext) into the datom structure. - r#"CREATE VIEW all_datoms AS - SELECT e, a, v, tx, value_type_tag, index_avet, index_vaet, index_fulltext, unique_value - FROM datoms - WHERE index_fulltext IS 0 - UNION ALL - SELECT e, a, v, tx, value_type_tag, index_avet, index_vaet, index_fulltext, unique_value - FROM fulltext_datoms"#, - - // Materialized views of the schema. - r#"CREATE TABLE idents (ident TEXT NOT NULL PRIMARY KEY, entid INTEGER UNIQUE NOT NULL)"#, - r#"CREATE TABLE schema (ident TEXT NOT NULL, attr TEXT NOT NULL, value BLOB NOT NULL, value_type_tag SMALLINT NOT NULL, - FOREIGN KEY (ident) REFERENCES idents (ident))"#, - r#"CREATE INDEX idx_schema_unique ON schema (ident, attr, value, value_type_tag)"#, - r#"CREATE TABLE parts (part TEXT NOT NULL PRIMARY KEY, start INTEGER NOT NULL, idx INTEGER NOT NULL)"#, - ]; - -/// Additional SQL statements to be executed, in order, to create the Mentat SQL schema (version 2). -/// We assume that the `V1_STATEMENTS` have been successfully executed (in order) before these are -/// executed. -#[cfg_attr(rustfmt, rustfmt_skip)] -const V2_STATEMENTS: [&'static str; 0] = []; +lazy_static! { + /// SQL statements to be executed, in order, to create the Mentat SQL schema (version 2). + #[cfg_attr(rustfmt, rustfmt_skip)] + static ref V2_STATEMENTS: Vec<&'static str> = { vec![ + r#"CREATE TABLE datoms (e INTEGER NOT NULL, a SMALLINT NOT NULL, v BLOB NOT NULL, tx INTEGER NOT NULL, + value_type_tag SMALLINT NOT NULL, + index_avet TINYINT NOT NULL DEFAULT 0, index_vaet TINYINT NOT NULL DEFAULT 0, + index_fulltext TINYINT NOT NULL DEFAULT 0, + unique_value TINYINT NOT NULL DEFAULT 0)"#, + r#"CREATE UNIQUE INDEX idx_datoms_eavt ON datoms (e, a, value_type_tag, v)"#, + r#"CREATE UNIQUE INDEX idx_datoms_aevt ON datoms (a, e, value_type_tag, v)"#, + + // Opt-in index: only if a has :db/index true. + r#"CREATE UNIQUE INDEX idx_datoms_avet ON datoms (a, value_type_tag, v, e) WHERE index_avet IS NOT 0"#, + + // Opt-in index: only if a has :db/valueType :db.type/ref. No need for tag here since all + // indexed elements are refs. + r#"CREATE UNIQUE INDEX idx_datoms_vaet ON datoms (v, a, e) WHERE index_vaet IS NOT 0"#, + + // Opt-in index: only if a has :db/fulltext true; thus, it has :db/valueType :db.type/string, + // which is not :db/valueType :db.type/ref. That is, index_vaet and index_fulltext are mutually + // exclusive. + r#"CREATE INDEX idx_datoms_fulltext ON datoms (value_type_tag, v, a, e) WHERE index_fulltext IS NOT 0"#, + + // TODO: possibly remove this index. :db.unique/{value,identity} should be asserted by the + // transactor in all cases, but the index may speed up some of SQLite's query planning. For now, + // it serves to validate the transactor implementation. Note that tag is needed here to + // differentiate, e.g., keywords and strings. + r#"CREATE UNIQUE INDEX idx_datoms_unique_value ON datoms (a, value_type_tag, v) WHERE unique_value IS NOT 0"#, + + r#"CREATE TABLE transactions (e INTEGER NOT NULL, a SMALLINT NOT NULL, v BLOB NOT NULL, tx INTEGER NOT NULL, added TINYINT NOT NULL DEFAULT 1, value_type_tag SMALLINT NOT NULL)"#, + r#"CREATE INDEX idx_transactions_tx ON transactions (tx, added)"#, + + // Fulltext indexing. + // A fulltext indexed value v is an integer rowid referencing fulltext_values. + + // Optional settings: + // tokenize="porter"#, + // prefix='2,3' + // By default we use Unicode-aware tokenizing (particularly for case folding), but preserve + // diacritics. + r#"CREATE VIRTUAL TABLE fulltext_values + USING FTS4 (text NOT NULL, searchid INT, tokenize=unicode61 "remove_diacritics=0")"#, + + // This combination of view and triggers allows you to transparently + // update-or-insert into FTS. Just INSERT INTO fulltext_values_view (text, searchid). + r#"CREATE VIEW fulltext_values_view AS SELECT * FROM fulltext_values"#, + r#"CREATE TRIGGER replace_fulltext_searchid + INSTEAD OF INSERT ON fulltext_values_view + WHEN EXISTS (SELECT 1 FROM fulltext_values WHERE text = new.text) + BEGIN + UPDATE fulltext_values SET searchid = new.searchid WHERE text = new.text; + END"#, + r#"CREATE TRIGGER insert_fulltext_searchid + INSTEAD OF INSERT ON fulltext_values_view + WHEN NOT EXISTS (SELECT 1 FROM fulltext_values WHERE text = new.text) + BEGIN + INSERT INTO fulltext_values (text, searchid) VALUES (new.text, new.searchid); + END"#, + + // A view transparently interpolating fulltext indexed values into the datom structure. + r#"CREATE VIEW fulltext_datoms AS + SELECT e, a, fulltext_values.text AS v, tx, value_type_tag, index_avet, index_vaet, index_fulltext, unique_value + FROM datoms, fulltext_values + WHERE datoms.index_fulltext IS NOT 0 AND datoms.v = fulltext_values.rowid"#, + + // A view transparently interpolating all entities (fulltext and non-fulltext) into the datom structure. + r#"CREATE VIEW all_datoms AS + SELECT e, a, v, tx, value_type_tag, index_avet, index_vaet, index_fulltext, unique_value + FROM datoms + WHERE index_fulltext IS 0 + UNION ALL + SELECT e, a, v, tx, value_type_tag, index_avet, index_vaet, index_fulltext, unique_value + FROM fulltext_datoms"#, + + // Materialized views of the schema. + r#"CREATE TABLE idents (ident TEXT NOT NULL PRIMARY KEY, entid INTEGER UNIQUE NOT NULL)"#, + r#"CREATE TABLE schema (ident TEXT NOT NULL, attr TEXT NOT NULL, value BLOB NOT NULL, value_type_tag SMALLINT NOT NULL, + FOREIGN KEY (ident) REFERENCES idents (ident))"#, + r#"CREATE INDEX idx_schema_unique ON schema (ident, attr, value, value_type_tag)"#, + r#"CREATE TABLE parts (part TEXT NOT NULL PRIMARY KEY, start INTEGER NOT NULL, idx INTEGER NOT NULL)"#, + ] + }; +} /// Set the SQLite user version. /// @@ -160,11 +154,7 @@ fn get_user_version(conn: &rusqlite::Connection) -> Result { pub fn create_current_version(conn: &mut rusqlite::Connection) -> Result { let tx = conn.transaction()?; - for statement in &V1_STATEMENTS { - tx.execute(statement, &[])?; - } - - for statement in &V2_STATEMENTS { + for statement in (&V2_STATEMENTS).iter() { try!(tx.execute(statement, &[])); } From aec7589300c96616ec2f03cf53f2aa1804dfc213 Mon Sep 17 00:00:00 2001 From: Nick Alexander Date: Mon, 23 Jan 2017 13:22:05 -0800 Subject: [PATCH 07/14] Review comment: Prefer ? to try!. --- db/src/db.rs | 22 +++++++++++----------- db/src/debug.rs | 2 +- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/db/src/db.rs b/db/src/db.rs index 46394ca80..a2fb38b8e 100644 --- a/db/src/db.rs +++ b/db/src/db.rs @@ -155,7 +155,7 @@ pub fn create_current_version(conn: &mut rusqlite::Connection) -> Result { let tx = conn.transaction()?; for statement in (&V2_STATEMENTS).iter() { - try!(tx.execute(statement, &[])); + tx.execute(statement, &[])?; } let bootstrap_partition_map = bootstrap::bootstrap_partition_map(); @@ -164,14 +164,14 @@ pub fn create_current_version(conn: &mut rusqlite::Connection) -> Result { // TODO: one insert, chunk into 999/3 sections, for safety. for (part, partition) in bootstrap_partition_map.iter() { // TODO: Convert "keyword" part to SQL using Value conversion. - try!(tx.execute("INSERT INTO parts VALUES (?, ?, ?)", &[part, &partition.start, &partition.index])); + tx.execute("INSERT INTO parts VALUES (?, ?, ?)", &[part, &partition.start, &partition.index])?; } let bootstrap_db = DB::new(bootstrap_partition_map, bootstrap::bootstrap_schema()); bootstrap_db.transact_internal(&tx, &bootstrap::bootstrap_entities()[..])?; - try!(set_user_version(&tx, CURRENT_VERSION)); - let user_version = try!(get_user_version(&tx)); + set_user_version(&tx, CURRENT_VERSION)?; + let user_version = get_user_version(&tx)?; // TODO: use the drop semantics to do this automagically? tx.commit()?; @@ -266,12 +266,12 @@ pub fn update_from_version(conn: &mut rusqlite::Connection, current_version: i32 bail!(ErrorKind::BadSQLiteStoreVersion(current_version)) } - let tx = try!(conn.transaction()); + let tx = conn.transaction()?; // TODO: actually implement upgrade. - try!(set_user_version(&tx, CURRENT_VERSION)); - let user_version = try!(get_user_version(&tx)); + set_user_version(&tx, CURRENT_VERSION)?; + let user_version = get_user_version(&tx)?; // TODO: use the drop semantics to do this automagically? - try!(tx.commit()); + tx.commit()?; Ok(user_version) } @@ -355,7 +355,7 @@ pub fn read_ident_map(conn: &rusqlite::Connection) -> Result { /// Read the partition map materialized view from the given SQL store. pub fn read_partition_map(conn: &rusqlite::Connection) -> Result { - let mut stmt: rusqlite::Statement = try!(conn.prepare("SELECT part, start, idx FROM parts")); + let mut stmt: rusqlite::Statement = conn.prepare("SELECT part, start, idx FROM parts")?; let m = stmt.query_and_then(&[], |row| -> Result<(String, Partition)> { Ok((row.get_checked(0)?, Partition::new(row.get_checked(1)?, row.get_checked(2)?))) })?.collect(); @@ -410,7 +410,7 @@ pub fn read_db(conn: &rusqlite::Connection) -> Result { // pub fn bootstrap(conn: &mut rusqlite::Connection, from_version: i32) -> Result<()> { // match from_version { // 0 => { -// try!(conn.execute(&format!("INSERT INTO parts VALUES = {}", "()"), &[][..])); +// conn.execute(&format!("INSERT INTO parts VALUES = {}", "()"), &[][..])?; // Ok(()) // }, // // 1 => { @@ -433,7 +433,7 @@ impl DB { v: entmod::ValueOrLookupRef::Value(ref v_), tx: _ } => { - let mut stmt: rusqlite::Statement = try!(conn.prepare("INSERT INTO datoms(e, a, v, tx, value_type_tag, index_avet, index_vaet, index_fulltext, unique_value) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)")); + let mut stmt: rusqlite::Statement = conn.prepare("INSERT INTO datoms(e, a, v, tx, value_type_tag, index_avet, index_vaet, index_fulltext, unique_value) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)")?; let e: i64 = *self.schema.get_entid(&e_.to_string()).ok_or(rusqlite::Error::InvalidColumnName(format!("Could not find entid for ident: {:?}", e_)))?; let a: i64 = *self.schema.get_entid(&a_.to_string()).ok_or(rusqlite::Error::InvalidColumnName(format!("Could not find entid for ident: {:?}", a_)))?; let attributes: &Attribute = self.schema.schema_map.get(&a).ok_or(rusqlite::Error::InvalidColumnName(format!("Could not find attributes for entid: {:?}", a)))?; diff --git a/db/src/debug.rs b/db/src/debug.rs index 6c583bad0..1ddf3859a 100644 --- a/db/src/debug.rs +++ b/db/src/debug.rs @@ -40,7 +40,7 @@ pub fn datoms(conn: &rusqlite::Connection, db: &DB) -> Result> { /// Return the set of datoms in the store with transaction ID strictly /// greater than the given `tx`, ordered by (tx, e, a, v). pub fn datoms_after(conn: &rusqlite::Connection, db: &DB, tx: &i32) -> Result> { - let mut stmt: rusqlite::Statement = try!(conn.prepare("SELECT e, a, v, value_type_tag FROM datoms WHERE tx > ? ORDER BY tx, e, a, v")); + let mut stmt: rusqlite::Statement = conn.prepare("SELECT e, a, v, value_type_tag FROM datoms WHERE tx > ? ORDER BY tx, e, a, v")?; // Convert numeric entid to entity Entid. let to_entid = |x| { From b1f8bd75754e0110cec6496d4b1fc81fed6765b4 Mon Sep 17 00:00:00 2001 From: Nick Alexander Date: Mon, 23 Jan 2017 13:28:46 -0800 Subject: [PATCH 08/14] Review comment: Fix typos; format; add TODOs. --- db/src/db.rs | 1 + db/src/types.rs | 21 ++++++++++++++++----- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/db/src/db.rs b/db/src/db.rs index a2fb38b8e..d679759a5 100644 --- a/db/src/db.rs +++ b/db/src/db.rs @@ -433,6 +433,7 @@ impl DB { v: entmod::ValueOrLookupRef::Value(ref v_), tx: _ } => { + // TODO: prepare and cache all these statements outside the transaction loop. let mut stmt: rusqlite::Statement = conn.prepare("INSERT INTO datoms(e, a, v, tx, value_type_tag, index_avet, index_vaet, index_fulltext, unique_value) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)")?; let e: i64 = *self.schema.get_entid(&e_.to_string()).ok_or(rusqlite::Error::InvalidColumnName(format!("Could not find entid for ident: {:?}", e_)))?; let a: i64 = *self.schema.get_entid(&a_.to_string()).ok_or(rusqlite::Error::InvalidColumnName(format!("Could not find entid for ident: {:?}", a_)))?; diff --git a/db/src/types.rs b/db/src/types.rs index 8df6bce36..1b3f39282 100644 --- a/db/src/types.rs +++ b/db/src/types.rs @@ -15,13 +15,13 @@ use std::collections::{BTreeMap}; /// Core types defining a Mentat knowledge base. /// /// At its core, Mentat maintains a set of assertions of the form entity-attribute-value (EAV). The -/// assertions conform to a schema where-by the given attribute constrains the associated value/set +/// assertions conform to a schema whereby the given attribute constrains the associated value/set /// of associated values. /// /// ## Assertions /// /// Mentat assertions are represented as rows in the `datoms` SQLite table, and each Mentat row -/// representing an assertion are tagged with a numeric representation of :db/valueType. +/// representing an assertion is with a numeric representation of :db/valueType. /// /// The tag is used to limit queries, and therefore is placed carefully in the relevant indices to /// allow searching numeric longs and doubles quickly. The tag is also used to convert SQLite @@ -45,7 +45,7 @@ use std::collections::{BTreeMap}; /// /// The entid sequence in a given partition is monotonically increasing, although not necessarily /// contiguous. That is, it is possible for a specific entid to have never been present in the -/// system, even though its predecessor and successor is present. +/// system, even though its predecessor and successor are present. // #[derive(Debug)] // pub enum Error { @@ -67,8 +67,8 @@ use std::collections::{BTreeMap}; /// use i64 rather than manually truncating u64 to u63 and casting to i64 throughout the codebase. pub type Entid = i64; -/// The attribute of each Mentat assertion has an :db/valueType constraining the value to a -/// particular domain. Mentat recognizes the following :db/valueType values. +/// The attribute of each Mentat assertion has a :db/valueType constraining the value to a +/// particular set. Mentat recognizes the following :db/valueType values. #[derive(Clone,Debug,Eq,Hash,Ord,PartialOrd,PartialEq)] pub enum ValueType { Ref, @@ -140,15 +140,18 @@ pub type PartitionMap = BTreeMap; pub struct Attribute { /// The associated value type, i.e., `:db/valueType`? pub value_type: ValueType, + /// `true` if this attribute is multi-valued, i.e., it is `:db/cardinality /// :db.cardinality/many`. `false` if this attribute is single-valued (the default), i.e., it /// is `:db/cardinality :db.cardinality/one`. pub multival: bool, + /// `true` if this attribute is unique-value, i.e., it is `:db/unique :db.unique/value`. /// /// *Unique-value* means that there is at most one assertion with the attribute and a /// particular value in the datom store. pub unique_value: bool, + /// `true` if this attribute is unique-identity, i.e., it is `:db/unique :db.unique/identity`. /// /// Unique-identity attributes always have value type `Ref`. @@ -156,12 +159,15 @@ pub struct Attribute { /// *Unique-identity* means that the attribute is *unique-value* and that they can be used in /// lookup-refs and will automatically upsert where appropriate. pub unique_identity: bool, + /// `true` if this attribute is automatically indexed, i.e., it is `:db/indexing true`. pub index: bool, + /// `true` if this attribute is automatically fulltext indexed, i.e., it is `:db/fulltext true`. /// /// Fulltext attributes always have string values. pub fulltext: bool, + /// `true` if this attribute is a component, i.e., it is `:db/isComponent true`. /// /// Component attributes always have value type `Ref`. @@ -208,10 +214,12 @@ pub struct Schema { /// /// Invariant: is the inverse map of `ident_map`. pub entid_map: EntidMap, + /// Map ident->entid. /// /// Invariant: is the inverse map of `entid_map`. pub ident_map: IdentMap, + /// Map entid->attribute flags. /// /// Invariant: key-set is the same as the key-set of `entid_map` (equivalently, the value-set of @@ -231,12 +239,15 @@ impl Schema { } /// Represents the metadata required to query from, or apply transactions to, a Mentat store. +/// +/// See https://github.com/mozilla/mentat/wiki/Thoughts:-modeling-db-conn-in-Rust. #[derive(Clone,Debug,Default,Eq,Hash,Ord,PartialOrd,PartialEq)] pub struct DB { /// Map partition name->`Partition`. /// /// TODO: represent partitions as entids. pub partition_map: PartitionMap, + /// The schema of the store. pub schema: Schema, } From 46c8fec5bbe6ae782848217172c99e24021f1bc3 Mon Sep 17 00:00:00 2001 From: Nick Alexander Date: Mon, 23 Jan 2017 14:43:44 -0800 Subject: [PATCH 09/14] Review comment: Assert that Mentat `Schema` is valid upon creation. --- db/src/db.rs | 2 +- db/src/schema.rs | 42 +++++++++++++++++++++++++++++++++++++----- db/src/types.rs | 11 ----------- 3 files changed, 38 insertions(+), 17 deletions(-) diff --git a/db/src/db.rs b/db/src/db.rs index d679759a5..a1a4ca33f 100644 --- a/db/src/db.rs +++ b/db/src/db.rs @@ -365,7 +365,7 @@ pub fn read_partition_map(conn: &rusqlite::Connection) -> Result { /// Read the schema materialized view from the given SQL store. pub fn read_schema(conn: &rusqlite::Connection, ident_map: &IdentMap) -> Result { // TODO: consider a less expensive way to do this inversion. - let schema_for_idents = Schema::new(ident_map.clone(), SchemaMap::default()); + let schema_for_idents = Schema::from(ident_map.clone(), SchemaMap::default())?; let mut stmt: rusqlite::Statement = conn.prepare("SELECT ident, attr, value, value_type_tag FROM schema")?; let r: Result> = stmt.query_and_then(&[], |row| { diff --git a/db/src/schema.rs b/db/src/schema.rs index 5a8660916..47da1a900 100644 --- a/db/src/schema.rs +++ b/db/src/schema.rs @@ -12,9 +12,31 @@ use edn::types::Value; use errors::*; -use types::{Attribute, Entid, IdentMap, Schema, SchemaMap, ValueType}; +use types::{Attribute, Entid, EntidMap, IdentMap, Schema, SchemaMap, ValueType}; use values; +/// Return `Ok(())` if `schema_map` defines a valid Mentat schema. +fn validate_schema_map(entid_map: &EntidMap, schema_map: &SchemaMap) -> Result<()> { + for (entid, attribute) in schema_map { + let ident = entid_map.get(entid).ok_or(ErrorKind::BadSchemaAssertion(format!("Could not get ident for entid: {}", entid)))?; + + if attribute.unique_identity && !attribute.unique_value { + bail!(ErrorKind::BadSchemaAssertion(format!(":db/unique :db/unique_identity without :db/unique :db/unique_value for entid: {}", ident))) + } + if attribute.fulltext && attribute.value_type != ValueType::String { + bail!(ErrorKind::BadSchemaAssertion(format!(":db/fulltext true without :db/valueType :db.type/string for entid: {}", ident))) + } + if attribute.component && attribute.value_type != ValueType::Ref { + bail!(ErrorKind::BadSchemaAssertion(format!(":db/isComponent true without :db/valueType :db.type/ref for entid: {}", ident))) + } + // TODO: consider warning if we have :db/index true for :db/valueType :db.type/string, + // since this may be inefficient. More generally, we should try to drive complex + // :db/valueType (string, uri, json in the future) users to opt-in to some hash-indexing + // scheme, as discussed in https://github.com/mozilla/mentat/issues/69. + } + Ok(()) +} + impl Schema { pub fn get_ident(&self, x: &Entid) -> Option<&String> { self.entid_map.get(x) @@ -28,6 +50,19 @@ impl Schema { self.schema_map.get(x) } + /// Create a valid `Schema` from the constituent maps. + pub fn from(ident_map: IdentMap, schema_map: SchemaMap) -> Result { + let entid_map: EntidMap = ident_map.iter().map(|(k, v)| (v.clone(), k.clone())).collect(); + + validate_schema_map(&entid_map, &schema_map)?; + + Ok(Schema { + ident_map: ident_map, + entid_map: entid_map, + schema_map: schema_map, + }) + } + /// Turn Value([[IDENT ATTR VALUE] ...]) into a Mentat `Schema`. pub fn from_ident_map_and_assertions(ident_map: IdentMap, assertions: &Value) -> Result { // Convert Value([[IDENT ATTR VALUE] ...]) to vec![(IDENT.to_string(), ATTR.to_string(), VALUE), ...]. @@ -88,7 +123,6 @@ impl Schema { } }, ":db/unique" => { - // TODO: assert that we're indexing? if *value == *values::DB_UNIQUE_VALUE { attributes.unique_value = true; } else if *value == *values::DB_UNIQUE_IDENTITY { @@ -108,7 +142,6 @@ impl Schema { } }, ":db/fulltext" => { - // TODO: check valueType is :db.type/string. if *value == Value::Boolean(true) { attributes.index = true; attributes.fulltext = true; @@ -119,7 +152,6 @@ impl Schema { } }, ":db/isComponent" => { - // TODO: check valueType is :db.type/ref. if *value == Value::Boolean(true) { attributes.component = true; } else if *value == Value::Boolean(false) { @@ -143,6 +175,6 @@ impl Schema { } }; - Ok(Schema::new(ident_map.clone(), schema_map)) + Schema::from(ident_map.clone(), schema_map) } } diff --git a/db/src/types.rs b/db/src/types.rs index 1b3f39282..b67c6210e 100644 --- a/db/src/types.rs +++ b/db/src/types.rs @@ -227,17 +227,6 @@ pub struct Schema { pub schema_map: SchemaMap, } -impl Schema { - pub fn new(ident_map: IdentMap, schema_map: SchemaMap) -> Schema { - let entid_map: EntidMap = ident_map.iter().map(|(k, v)| (v.clone(), k.clone())).collect(); - Schema { - ident_map: ident_map, - entid_map: entid_map, - schema_map: schema_map, - } - } -} - /// Represents the metadata required to query from, or apply transactions to, a Mentat store. /// /// See https://github.com/mozilla/mentat/wiki/Thoughts:-modeling-db-conn-in-Rust. From 4bd911efb4de9d41bc1e0b27d3b6b9f576f8bab3 Mon Sep 17 00:00:00 2001 From: Nick Alexander Date: Wed, 25 Jan 2017 10:32:50 -0800 Subject: [PATCH 10/14] Review comment: Improve conversion to and from SQL values. This patch factors the fundamental SQL conversion maps between (rusqlite::Value, value_type_tag) and (edn::Value, ValueType) through a new Mentat TypedValue. (A future patch might rename this fundamental type mentat::Value.) To make certain conversion functions infallible, I removed placeholders for :db.type/{instant,uuid,uri}. (We could panic instead, but there's no need to do that right now.) --- .travis.yml | 1 + Cargo.toml | 2 +- db/Cargo.toml | 15 +-- db/src/bootstrap.rs | 134 +++++++++++++++------- db/src/db.rs | 262 +++++++++++++++++++++----------------------- db/src/debug.rs | 7 +- db/src/entids.rs | 58 ++++++++++ db/src/errors.rs | 73 ++++-------- db/src/lib.rs | 3 + db/src/schema.rs | 164 +++++++++++++-------------- db/src/types.rs | 64 ++++------- 11 files changed, 408 insertions(+), 375 deletions(-) create mode 100644 db/src/entids.rs diff --git a/.travis.yml b/.travis.yml index 4be525701..70ad62628 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,7 @@ script: - cargo build --verbose - cargo test --verbose - cargo test --verbose -p edn + - cargo test --verbose -p mentat_db - cargo test --verbose -p mentat_query - cargo test --verbose -p mentat_query_parser - cargo test --verbose -p mentat_tx_parser diff --git a/Cargo.toml b/Cargo.toml index 5d0082b03..8d2f557a6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,7 +6,7 @@ authors = ["Richard Newman ", "Nicholas Alexander = { - vec![(":db/ident", 1), - (":db.part/db", 2), - (":db/txInstant", 3), - (":db.install/partition", 4), - (":db.install/valueType", 5), - (":db.install/attribute", 6), - (":db/valueType", 7), - (":db/cardinality", 8), - (":db/unique", 9), - (":db/isComponent", 10), - (":db/index", 11), - (":db/fulltext", 12), - (":db/noHistory", 13), - (":db/add", 14), - (":db/retract", 15), - (":db.part/user", 16), - (":db.part/tx", 17), - (":db/excise", 18), - (":db.excise/attrs", 19), - (":db.excise/beforeT", 20), - (":db.excise/before", 21), - (":db.alter/attribute", 22), - (":db.type/ref", 23), - (":db.type/keyword", 24), - (":db.type/long", 25), - (":db.type/double", 26), - (":db.type/string", 27), - (":db.type/boolean", 28), - (":db.type/instant", 29), - (":db.type/bytes", 30), - (":db.cardinality/one", 31), - (":db.cardinality/many", 32), - (":db.unique/value", 33), - (":db.unique/identity", 34), - (":db/doc", 35), + vec![(":db/ident", entids::DB_IDENT), + (":db.part/db", entids::DB_PART_DB), + (":db/txInstant", entids::DB_TX_INSTANT), + (":db.install/partition", entids::DB_INSTALL_PARTITION), + (":db.install/valueType", entids::DB_INSTALL_VALUETYPE), + (":db.install/attribute", entids::DB_INSTALL_ATTRIBUTE), + (":db/valueType", entids::DB_VALUE_TYPE), + (":db/cardinality", entids::DB_CARDINALITY), + (":db/unique", entids::DB_UNIQUE), + (":db/isComponent", entids::DB_IS_COMPONENT), + (":db/index", entids::DB_INDEX), + (":db/fulltext", entids::DB_FULLTEXT), + (":db/noHistory", entids::DB_NO_HISTORY), + (":db/add", entids::DB_ADD), + (":db/retract", entids::DB_RETRACT), + (":db.part/user", entids::DB_PART_USER), + (":db.part/tx", entids::DB_PART_TX), + (":db/excise", entids::DB_EXCISE), + (":db.excise/attrs", entids::DB_EXCISE_ATTRS), + (":db.excise/beforeT", entids::DB_EXCISE_BEFORET), + (":db.excise/before", entids::DB_EXCISE_BEFORE), + (":db.alter/attribute", entids::DB_ALTER_ATTRIBUTE), + (":db.type/ref", entids::DB_TYPE_REF), + (":db.type/keyword", entids::DB_TYPE_KEYWORD), + (":db.type/long", entids::DB_TYPE_LONG), + (":db.type/double", entids::DB_TYPE_DOUBLE), + (":db.type/string", entids::DB_TYPE_STRING), + (":db.type/boolean", entids::DB_TYPE_BOOLEAN), + (":db.type/instant", entids::DB_TYPE_INSTANT), + (":db.type/bytes", entids::DB_TYPE_BYTES), + (":db.cardinality/one", entids::DB_CARDINALITY_ONE), + (":db.cardinality/many", entids::DB_CARDINALITY_MANY), + (":db.unique/value", entids::DB_UNIQUE_VALUE), + (":db.unique/identity", entids::DB_UNIQUE_IDENTITY), + (":db/doc", entids::DB_DOC), ] }; static ref V2_IDENTS: Vec<(&'static str, i64)> = { [(*V1_IDENTS).clone(), - vec![(":db.schema/version", 36), - (":db.schema/attribute", 37), + vec![(":db.schema/version", entids::DB_SCHEMA_VERSION), + (":db.schema/attribute", entids::DB_SCHEMA_ATTRIBUTE), ]].concat() }; @@ -150,6 +151,57 @@ fn idents_to_assertions(idents: &[(&str, i64)]) -> Vec { .collect() } +/// Convert {:ident {:key :value ...} ...} to vec![(String(:ident), String(:key), TypedValue(:value)), ...]. +/// +/// Such triples are closer to what the transactor will produce when processing +/// :db.install/attribute assertions. +fn symbolic_schema_to_triples(ident_map: &IdentMap, symbolic_schema: &Value) -> Result> { + // Failure here is a coding error, not a runtime error. + let mut triples: Vec<(String, String, TypedValue)> = vec![]; + // TODO: Consider `flat_map` and `map` rather than loop. + match *symbolic_schema { + Value::Map(ref m) => { + for (ident, mp) in m { + let ident = match ident { + &Value::NamespacedKeyword(ref ident) => ident.to_string(), + _ => bail!(ErrorKind::BadBootstrapDefinition(format!("Expected namespaced keyword for ident but got '{:?}'", ident))) + }; + match *mp { + Value::Map(ref mpp) => { + for (attr, value) in mpp { + let attr = match attr { + &Value::NamespacedKeyword(ref attr) => attr.to_string(), + _ => bail!(ErrorKind::BadBootstrapDefinition(format!("Expected namespaced keyword for attr but got '{:?}'", attr))) + }; + + // We have symbolic idents but the transactor handles entids. Ad-hoc + // convert right here. This is a fundamental limitation on the + // bootstrap symbolic schema format; we can't represent "real" keywords + // at this time. + // + // TODO: remove this limitation, perhaps by including a type tag in the + // bootstrap symbolic schema, or by representing the initial bootstrap + // schema directly as Rust data. + let typed_value = match TypedValue::from_edn_value(value) { + Some(TypedValue::Keyword(ref s)) => TypedValue::Ref(*ident_map.get(s).ok_or(ErrorKind::UnrecognizedIdent(s.clone()))?), + Some(v) => v, + _ => bail!(ErrorKind::BadBootstrapDefinition(format!("Expected Mentat typed value for value but got '{:?}'", value))) + }; + + triples.push((ident.clone(), + attr.clone(), + typed_value)); + } + }, + _ => bail!(ErrorKind::BadBootstrapDefinition("Expected {:db/ident {:db/attr value ...} ...}".into())) + } + } + }, + _ => bail!(ErrorKind::BadBootstrapDefinition("Expected {...}".into())) + } + Ok(triples) +} + /// Convert {IDENT {:key :value ...} ...} to [[:db/add IDENT :key :value] ...]. /// In addition, add [:db.add :db.part/db :db.install/attribute IDENT] installation assertions. fn symbolic_schema_to_assertions(symbolic_schema: &Value) -> Result> { @@ -193,9 +245,9 @@ pub fn bootstrap_ident_map() -> IdentMap { } pub fn bootstrap_schema() -> Schema { - let bootstrap_assertions: Value = Value::Vector(symbolic_schema_to_assertions(&V2_SYMBOLIC_SCHEMA).unwrap()); - Schema::from_ident_map_and_assertions(bootstrap_ident_map(), &bootstrap_assertions) - .unwrap() + let ident_map = bootstrap_ident_map(); + let bootstrap_triples = symbolic_schema_to_triples(&ident_map, &V2_SYMBOLIC_SCHEMA).unwrap(); + Schema::from_ident_map_and_triples(ident_map, bootstrap_triples).unwrap() } pub fn bootstrap_entities() -> Vec { diff --git a/db/src/db.rs b/db/src/db.rs index a1a4ca33f..5d3ba8856 100644 --- a/db/src/db.rs +++ b/db/src/db.rs @@ -19,8 +19,6 @@ use errors::*; use mentat_tx::entities as entmod; use mentat_tx::entities::Entity; use types::*; -use values; -use {to_namespaced_keyword}; pub fn new_connection() -> rusqlite::Connection { return rusqlite::Connection::open_in_memory().unwrap(); @@ -285,60 +283,62 @@ pub fn ensure_current_version(conn: &mut rusqlite::Connection) -> Result { } } -/// Given a SQLite `value` and a `value_type_tag`, return the corresponding EDN `Value`. -pub fn to_edn(value: &rusqlite::types::Value, value_type_tag: &i32) -> Result { - ValueType::from_value_type_tag(value_type_tag) - .ok_or(ErrorKind::BadValueTypeTag(*value_type_tag).into()) - .and_then(|value_type| -> Result { - match (value_type, value) { - (ValueType::Ref, &rusqlite::types::Value::Integer(ref x)) => Ok(Value::Integer(*x)), - (ValueType::Boolean, &rusqlite::types::Value::Integer(ref x)) => Ok(Value::Boolean(0 != *x)), - (ValueType::Instant, &rusqlite::types::Value::Integer(_)) => bail!(ErrorKind::NotYetImplemented(":db.type/instant".into())), - // SQLite distinguishes integral from decimal types, allowing long and double to - // share a tag. That means the `value_type` above might be incorrect: we could see - // a `Long` tag and really have `Double` data. We always trust the tag: Mentat - // should be managing its metadata correctly. - (ValueType::Long, &rusqlite::types::Value::Integer(ref x)) => Ok(Value::Integer(*x)), - (ValueType::Long, &rusqlite::types::Value::Real(ref x)) if value_type_tag == &ValueType::Double.value_type_tag() => Ok(Value::Float((*x).into())), - // This is spurious, since `value_type` never returns `Double`, but let's just - // leave it in case we change things. - (ValueType::Double, &rusqlite::types::Value::Real(ref x)) => Ok(Value::Float((*x).into())), - (ValueType::String, &rusqlite::types::Value::Text(ref x)) => Ok(Value::Text(x.clone())), - (ValueType::UUID, &rusqlite::types::Value::Text(_)) => bail!(ErrorKind::NotYetImplemented(":db.type/uuid".into())), - (ValueType::URI, &rusqlite::types::Value::Text(_)) => bail!(ErrorKind::NotYetImplemented(":db.type/uri".into())), - (ValueType::Keyword, &rusqlite::types::Value::Text(ref x)) => Ok(Value::Text(x.clone())), - (_, value) => bail!(ErrorKind::BadValueAndTagPair(value.clone(), *value_type_tag)), - } - }) -} +impl TypedValue { + /// Given a SQLite `value` and a `value_type_tag`, return the corresponding `TypedValue`. + pub fn from_sql_value_pair(value: &rusqlite::types::Value, value_type_tag: &i32) -> Result { + match (*value_type_tag, value) { + (0, &rusqlite::types::Value::Integer(ref x)) => Ok(TypedValue::Ref(*x)), + (1, &rusqlite::types::Value::Integer(ref x)) => Ok(TypedValue::Boolean(0 != *x)), + // SQLite distinguishes integral from decimal types, allowing long and double to + // share a tag. + (5, &rusqlite::types::Value::Integer(ref x)) => Ok(TypedValue::Long(*x)), + (5, &rusqlite::types::Value::Real(ref x)) => Ok(TypedValue::Double((*x).into())), + (10, &rusqlite::types::Value::Text(ref x)) => Ok(TypedValue::String(x.clone())), + (13, &rusqlite::types::Value::Text(ref x)) => Ok(TypedValue::Keyword(x.clone())), + (_, value) => bail!(ErrorKind::BadSQLValuePair(value.clone(), *value_type_tag)), + } + } -/// A `Value` that can be converted `ToSql`. -/// -/// This is just working around Rust's refusal to allow us to implement a trait we don't originate -/// for a type we don't originate. -/// -/// TODO: &Value for efficiency. -struct SqlValueWrapper(Value); - -impl ToSql for SqlValueWrapper { - fn to_sql(&self) -> rusqlite::Result { - match self.0 { - Value::Nil => Ok(ToSqlOutput::from(rusqlite::types::Value::Null)), - Value::Boolean(x) => Ok(ToSqlOutput::from(x)), - Value::Integer(x) => Ok(ToSqlOutput::from(x)), - // TODO: consider using a larger radix to save characters. - Value::BigInteger(ref x) => Ok(ToSqlOutput::from(x.to_str_radix(10))), - Value::Float(ref x) => Ok(ToSqlOutput::from(x.into_inner())), - // TODO: try to avoid this clone. - Value::Text(ref x) => Ok(ToSqlOutput::from(x.clone())), - Value::PlainSymbol(ref x) => Ok(ToSqlOutput::from(x.to_string())), - Value::NamespacedSymbol(ref x) => Ok(ToSqlOutput::from(x.to_string())), - Value::Keyword(ref x) => Ok(ToSqlOutput::from(x.to_string())), - Value::NamespacedKeyword(ref x) => Ok(ToSqlOutput::from(x.to_string())), - Value::Vector(_) => Err(rusqlite::Error::InvalidColumnName(format!("Cannot convert to_sql: {:?}", self.0))), - Value::List(_) => Err(rusqlite::Error::InvalidColumnName(format!("Cannot convert to_sql: {:?}", self.0))), - Value::Set(_) => Err(rusqlite::Error::InvalidColumnName(format!("Cannot convert to_sql: {:?}", self.0))), - Value::Map(_) => Err(rusqlite::Error::InvalidColumnName(format!("Cannot convert to_sql: {:?}", self.0))), + /// Given an EDN `value`, return a corresponding Mentat `TypedValue`. + /// + /// An EDN `Value` does not encode a unique Mentat `ValueType`, so the composition + /// `from_edn_value(first(to_edn_value_pair(...)))` loses information. Additionally, there are + /// EDN values which are not Mentat typed values. + /// + /// This function is deterministic. + pub fn from_edn_value(value: &Value) -> Option { + match value { + &Value::Boolean(x) => Some(TypedValue::Boolean(x)), + &Value::Integer(x) => Some(TypedValue::Long(x)), + &Value::Float(ref x) => Some(TypedValue::Double(x.clone())), + &Value::Text(ref x) => Some(TypedValue::String(x.clone())), + &Value::NamespacedKeyword(ref x) => Some(TypedValue::Keyword(x.to_string())), + _ => None + } + } + + /// Return the corresponding SQLite `value` and `value_type_tag` pair. + pub fn to_sql_value_pair<'a>(&'a self) -> (ToSqlOutput<'a>, i32) { + match self { + &TypedValue::Ref(x) => (rusqlite::types::Value::Integer(x).into(), 0), + &TypedValue::Boolean(x) => (rusqlite::types::Value::Integer(if x { 1 } else { 0 }).into(), 1), + // SQLite distinguishes integral from decimal types, allowing long and double to share a tag. + &TypedValue::Long(x) => (rusqlite::types::Value::Integer(x).into(), 5), + &TypedValue::Double(x) => (rusqlite::types::Value::Real(x.into_inner()).into(), 5), + &TypedValue::String(ref x) => (rusqlite::types::ValueRef::Text(x.as_str()).into(), 10), + &TypedValue::Keyword(ref x) => (rusqlite::types::ValueRef::Text(x.as_str()).into(), 13), + } + } + + /// Return the corresponding EDN `value` and `value_type` pair. + pub fn to_edn_value_pair(&self) -> (Value, ValueType) { + match self { + &TypedValue::Ref(x) => (Value::Integer(x), ValueType::Ref), + &TypedValue::Boolean(x) => (Value::Boolean(x), ValueType::Boolean), + &TypedValue::Long(x) => (Value::Integer(x), ValueType::Long), + &TypedValue::Double(x) => (Value::Float(x), ValueType::Double), + &TypedValue::String(ref x) => (Value::Text(x.clone()), ValueType::String), + &TypedValue::Keyword(ref x) => (Value::Text(x.clone()), ValueType::Keyword), } } } @@ -364,38 +364,21 @@ pub fn read_partition_map(conn: &rusqlite::Connection) -> Result { /// Read the schema materialized view from the given SQL store. pub fn read_schema(conn: &rusqlite::Connection, ident_map: &IdentMap) -> Result { - // TODO: consider a less expensive way to do this inversion. - let schema_for_idents = Schema::from(ident_map.clone(), SchemaMap::default())?; - let mut stmt: rusqlite::Statement = conn.prepare("SELECT ident, attr, value, value_type_tag FROM schema")?; - let r: Result> = stmt.query_and_then(&[], |row| { + let r: Result> = stmt.query_and_then(&[], |row| { // Each row looks like :db/index|:db/valueType|28|0. Observe that 28|0 represents a - // :db.type/ref to entid 28, which needs to be converted to an ident. + // :db.type/ref to entid 28, which needs to be converted to a TypedValue. + // TODO: don't use textual ident and attr; just use entids directly. let symbolic_ident: String = row.get_checked(0)?; - let attr: String = row.get_checked(1)?; + let symbolic_attr: String = row.get_checked(1)?; let v: rusqlite::types::Value = row.get_checked(2)?; - let value_type_tag = row.get_checked(3)?; - - // We want a symbolic schema, but most of our values are :db.type/ref attributes. Map those - // entids back to idents. This is ad-hoc since we haven't yet a functional DB instance. - let value = match to_edn(&v, &value_type_tag)? { - Value::Integer(entid) if value_type_tag == ValueType::Ref.value_type_tag() => { - schema_for_idents.get_ident(&entid) - .and_then(|x| to_namespaced_keyword(&x)) - .map(Value::NamespacedKeyword) - .ok_or(rusqlite::Error::InvalidColumnName(format!("Could not map :db.type/ref {} to ident!", entid)))? - }, - value => value - }; - - Ok(Value::Vector(vec![ - values::DB_ADD.clone(), - to_namespaced_keyword(&symbolic_ident).map(Value::NamespacedKeyword).ok_or(rusqlite::Error::InvalidColumnName(format!("XX1!")))?, - to_namespaced_keyword(&attr).map(Value::NamespacedKeyword).ok_or(rusqlite::Error::InvalidColumnName(format!("XX2!")))?, - value])) + let value_type_tag: i32 = row.get_checked(3)?; + let typed_value = TypedValue::from_sql_value_pair(&v, &value_type_tag)?; + + Ok((symbolic_ident, symbolic_attr, typed_value)) })?.collect(); - r.and_then(|values| Schema::from_ident_map_and_assertions(ident_map.clone(), &Value::Vector(values))) + r.and_then(|triples| Schema::from_ident_map_and_triples(ident_map.clone(), triples)) } /// Read the materialized views from the given SQL store and return a Mentat `DB` for querying and @@ -407,20 +390,32 @@ pub fn read_db(conn: &rusqlite::Connection) -> Result { Ok(DB::new(partition_map, schema)) } -// pub fn bootstrap(conn: &mut rusqlite::Connection, from_version: i32) -> Result<()> { -// match from_version { -// 0 => { -// conn.execute(&format!("INSERT INTO parts VALUES = {}", "()"), &[][..])?; -// Ok(()) -// }, -// // 1 => { -// // }, -// // TODO: find a better error type for this. -// _ => Err(rusqlite::Error::InvalidColumnName(format!("Cannot handle bootstrapping from version {}!", from_version))) -// } -// } - impl DB { + /// Do schema-aware typechecking and coercion. + /// + /// Either assert that the given value is in the attribute's value set, or (in limited cases) + /// coerce the given value into the attribute's value set. + pub fn to_typed_value(&self, value: &Value, attribute: &Attribute) -> Result { + // TODO: encapsulate entid-ident-attribute for better error messages. + match TypedValue::from_edn_value(value) { + // We don't recognize this EDN at all. Get out! + None => bail!(ErrorKind::BadEDNValuePair(value.clone(), attribute.value_type.clone())), + Some(typed_value) => match (&attribute.value_type, typed_value) { + // Most types don't coerce at all. + (&ValueType::Boolean, tv @ TypedValue::Boolean(_)) => Ok(tv), + (&ValueType::Long, tv @ TypedValue::Long(_)) => Ok(tv), + (&ValueType::Double, tv @ TypedValue::Double(_)) => Ok(tv), + (&ValueType::String, tv @ TypedValue::String(_)) => Ok(tv), + (&ValueType::Keyword, tv @ TypedValue::Keyword(_)) => Ok(tv), + // Ref coerces a little: we interpret some things depending on the schema as a Ref. + (&ValueType::Ref, TypedValue::Long(x)) => Ok(TypedValue::Ref(x)), + (&ValueType::Ref, TypedValue::Keyword(ref x)) => self.schema.require_entid(&x.to_string()).map(|&entid| TypedValue::Ref(entid)), + // Otherwise, we have a type mismatch. + (value_type, _) => bail!(ErrorKind::BadEDNValuePair(value.clone(), value_type.clone())), + } + } + } + // TODO: move this to the transactor layer. pub fn transact_internal(&self, conn: &rusqlite::Connection, entities: &[Entity]) -> Result<()>{ // TODO: manage :db/tx, write :db/txInstant. @@ -434,35 +429,27 @@ impl DB { tx: _ } => { // TODO: prepare and cache all these statements outside the transaction loop. + // XXX: Error types. let mut stmt: rusqlite::Statement = conn.prepare("INSERT INTO datoms(e, a, v, tx, value_type_tag, index_avet, index_vaet, index_fulltext, unique_value) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)")?; - let e: i64 = *self.schema.get_entid(&e_.to_string()).ok_or(rusqlite::Error::InvalidColumnName(format!("Could not find entid for ident: {:?}", e_)))?; - let a: i64 = *self.schema.get_entid(&a_.to_string()).ok_or(rusqlite::Error::InvalidColumnName(format!("Could not find entid for ident: {:?}", a_)))?; - let attributes: &Attribute = self.schema.schema_map.get(&a).ok_or(rusqlite::Error::InvalidColumnName(format!("Could not find attributes for entid: {:?}", a)))?; - - let value_type_tag = attributes.value_type.value_type_tag(); - - // We can have [:db/ident :db/ref-attr :db/kw] and need to convert the ident - // :db/kw to an entid. So we need some initial type-checking. - // TODO: do the type-checking and value-mapping comprehensively. - let mut v__ = v_.clone(); - if attributes.value_type == ValueType::Ref { - match *v_ { - Value::Integer(_) => (), - Value::NamespacedKeyword(ref s) => { v__ = self.schema.get_entid(&s.to_string()).map(|&x| Value::Integer(x)).ok_or(rusqlite::Error::InvalidColumnName(format!("Could not find entid for ident: {:?}", e_)))? }, - _ => panic!("bad type"), - } - } - - // TODO: avoid spurious clone. - let v = SqlValueWrapper(v__.clone()); + let e: i64 = *self.schema.require_entid(&e_.to_string())?; + let a: i64 = *self.schema.require_entid(&a_.to_string())?; + let attribute: &Attribute = self.schema.require_attribute_for_entid(&a)?; + + // This is our chance to do schema-aware typechecking: to either assert that the + // given value is in the attribute's value set, or (in limited cases) to coerce + // the value into the attribute's value set. + let typed_value: TypedValue = self.to_typed_value(v_, &attribute)?; + + // Now we can represent the typed value as an SQL value. + let (value, value_type_tag): (ToSqlOutput, i32) = typed_value.to_sql_value_pair(); // Fun times, type signatures. - let values: [&ToSql; 9] = [&e, &a, &v, &tx, &value_type_tag, &attributes.index, to_bool_ref(attributes.value_type == ValueType::Ref), &attributes.fulltext, &attributes.unique_value]; + let values: [&ToSql; 9] = [&e, &a, &value, &tx, &value_type_tag, &attribute.index, to_bool_ref(attribute.value_type == ValueType::Ref), &attribute.fulltext, &attribute.unique_value]; stmt.insert(&values[..])?; Ok(()) }, // TODO: find a better error type for this. - _ => panic!(format!("Transacting entity not yet supported: {:?}", entity)) // rusqlite::Error::InvalidColumnName(format!("Not yet implented: entities of form ..."))) + _ => panic!(format!("Transacting entity not yet supported: {:?}", entity)) } }).collect(); @@ -476,37 +463,34 @@ mod tests { use super::*; use bootstrap; use debug; + use rusqlite; use types::*; - // #[test] - // fn test_open_current_version() { - // let mut conn = rusqlite::Connection::open("file:///Users/nalexander/Mozilla/mentat/fixtures/v2empty.db").unwrap(); - // // assert_eq!(ensure_current_version(&mut conn).unwrap(), CURRENT_VERSION); - - // // let mut map = IdentMap::new(); - // // assert_eq!(map.insert("a".into(), 1), None); + #[test] + fn test_open_current_version() { + // TODO: figure out how to reference the fixtures directory for real. For now, assume we're + // executing `cargo test` in `db/`. + let conn = rusqlite::Connection::open("../fixtures/v2empty.db").unwrap(); + // assert_eq!(ensure_current_version(&mut conn).unwrap(), CURRENT_VERSION); - // let partition_map = read_partition_map(&conn).unwrap(); - // // assert_eq!(partition_map, bootstrap_partition_map()); + // TODO: write :db/txInstant, bump :db.part/tx. + // let partition_map = read_partition_map(&conn).unwrap(); + // assert_eq!(partition_map, bootstrap::bootstrap_partition_map()); - // let ident_map = read_ident_map(&conn).unwrap(); - // assert_eq!(ident_map, bootstrap_ident_map()); + let ident_map = read_ident_map(&conn).unwrap(); + assert_eq!(ident_map, bootstrap::bootstrap_ident_map()); - // let schema = read_schema(&conn, &ident_map).unwrap(); - // assert_eq!(schema, Schema::default()); + let schema = read_schema(&conn, &ident_map).unwrap(); + assert_eq!(schema, bootstrap::bootstrap_schema()); // Schema::default()); - // let db = DB { - // partition_map: partition_map, - // schema: schema, - // }; + let db = read_db(&conn).unwrap(); - // // assert_eq!(ident_map, IdentMap::new()); - // // assert_eq!(read_partition_map(&conn).unwrap(), PartitionMap::new()); - // // assert_eq!(read_schema(&conn, &ident_map).unwrap(), Schema::default()); + let datoms = debug::datoms_after(&conn, &db, &0).unwrap(); + assert_eq!(datoms.len(), 89); // The 89th is the :db/txInstant value. - // // TODO: fewer magic numbers! - // assert_eq!(datoms_after(&conn, &db, &0x10000000).unwrap(), vec![]); - // } + // // TODO: fewer magic numbers! + // assert_eq!(debug::datoms_after(&conn, &db, &0x10000001).unwrap(), vec![]); + } #[test] fn test_create_current_version() { diff --git a/db/src/debug.rs b/db/src/debug.rs index 1ddf3859a..20c1be25f 100644 --- a/db/src/debug.rs +++ b/db/src/debug.rs @@ -17,8 +17,7 @@ use rusqlite; use {to_namespaced_keyword}; use edn::types::{Value}; use mentat_tx::entities::{Entid}; -use types::{DB}; -use db::{to_edn}; +use types::{DB, TypedValue}; use errors::Result; /// Represents an assertion (*datom*) in the store. @@ -52,7 +51,9 @@ pub fn datoms_after(conn: &rusqlite::Connection, db: &DB, tx: &i32) -> Resultentid mapping failed. + UnrecognizedIdent(ident: String) { + description("no entid found for ident") + display("no entid found for ident: '{}'", ident) + } + + /// An entid->ident mapping failed. + UnrecognizedEntid(entid: Entid) { + description("no ident found for entid") + display("no ident found for entid: '{}'", entid) + } } } diff --git a/db/src/lib.rs b/db/src/lib.rs index bc5498b5a..29fa7bf6d 100644 --- a/db/src/lib.rs +++ b/db/src/lib.rs @@ -12,6 +12,8 @@ extern crate error_chain; #[macro_use] extern crate lazy_static; +extern crate num; +extern crate ordered_float; extern crate rusqlite; extern crate edn; @@ -25,6 +27,7 @@ pub use types::*; pub mod db; mod bootstrap; mod debug; +mod entids; mod errors; mod schema; mod types; diff --git a/db/src/schema.rs b/db/src/schema.rs index 47da1a900..abf460126 100644 --- a/db/src/schema.rs +++ b/db/src/schema.rs @@ -10,10 +10,9 @@ #![allow(dead_code)] -use edn::types::Value; +use entids; use errors::*; -use types::{Attribute, Entid, EntidMap, IdentMap, Schema, SchemaMap, ValueType}; -use values; +use types::{Attribute, Entid, EntidMap, IdentMap, Schema, SchemaMap, TypedValue, ValueType}; /// Return `Ok(())` if `schema_map` defines a valid Mentat schema. fn validate_schema_map(entid_map: &EntidMap, schema_map: &SchemaMap) -> Result<()> { @@ -50,6 +49,18 @@ impl Schema { self.schema_map.get(x) } + pub fn require_ident(&self, entid: &Entid) -> Result<&String> { + self.get_ident(&entid).ok_or(ErrorKind::UnrecognizedEntid(*entid).into()) + } + + pub fn require_entid(&self, ident: &String) -> Result<&Entid> { + self.get_entid(&ident).ok_or(ErrorKind::UnrecognizedIdent(ident.clone()).into()) + } + + pub fn require_attribute_for_entid(&self, entid: &Entid) -> Result<&Attribute> { + self.attribute_for_entid(entid).ok_or(ErrorKind::UnrecognizedEntid(*entid).into()) + } + /// Create a valid `Schema` from the constituent maps. pub fn from(ident_map: IdentMap, schema_map: SchemaMap) -> Result { let entid_map: EntidMap = ident_map.iter().map(|(k, v)| (v.clone(), k.clone())).collect(); @@ -63,112 +74,85 @@ impl Schema { }) } - /// Turn Value([[IDENT ATTR VALUE] ...]) into a Mentat `Schema`. - pub fn from_ident_map_and_assertions(ident_map: IdentMap, assertions: &Value) -> Result { - // Convert Value([[IDENT ATTR VALUE] ...]) to vec![(IDENT.to_string(), ATTR.to_string(), VALUE), ...]. - let triples: Vec<(String, String, &Value)> = match *assertions { - Value::Vector(ref datoms) => { - datoms.into_iter().map(|datom| { - match datom { - &Value::Vector(ref values) => { - let mut i = values.iter(); - match (i.next(), i.next(), i.next(), i.next(), i.next()) { - (Some(add), Some(&Value::NamespacedKeyword(ref ident)), Some(&Value::NamespacedKeyword(ref attr)), Some(value), None) if *add == *values::DB_ADD => - Ok((ident.to_string(), attr.to_string(), value)), - _ => Err(ErrorKind::BadSchemaAssertion(format!("Expected [[:db/add IDENT ATTR VALUE] ...], got: {:?}", datom))) - } - }, - _ => Err(ErrorKind::BadSchemaAssertion(format!("Expected [[...] ...], got: {:?}", datom))) - } - }).collect() - }, - _ => Err(ErrorKind::BadSchemaAssertion(format!("Expected [...], got: {:?}", assertions))) - }?; - + /// Turn vec![(String(:ident), String(:key), TypedValue(:value)), ...] into a Mentat `Schema`. + pub fn from_ident_map_and_triples(ident_map: IdentMap, assertions: U) -> Result + where U: IntoIterator{ let mut schema_map = SchemaMap::new(); - for (ident, attr, value) in triples { - let entid: &i64 = ident_map.get(&ident).ok_or(ErrorKind::BadSchemaAssertion(format!("Could not get ")))?; - let attributes = schema_map.entry(*entid).or_insert(Attribute::default()); - - // Yes, this is pretty bonkers. Suggestions appreciated. - match attr.as_str() { - ":db/valueType" => { - if *value == *values::DB_TYPE_REF { - attributes.value_type = ValueType::Ref; - } else if *value == *values::DB_TYPE_BOOLEAN { - attributes.value_type = ValueType::Boolean; - } else if *value == *values::DB_TYPE_INSTANT { - attributes.value_type = ValueType::Instant; - } else if *value == *values::DB_TYPE_LONG { - attributes.value_type = ValueType::Long; - } else if *value == *values::DB_TYPE_STRING { - attributes.value_type = ValueType::String; - } else if *value == *values::DB_TYPE_UUID { - attributes.value_type = ValueType::UUID; - } else if *value == *values::DB_TYPE_URI { - attributes.value_type = ValueType::URI; - } else if *value == *values::DB_TYPE_KEYWORD { - attributes.value_type = ValueType::Keyword; - } else { - bail!(ErrorKind::BadSchemaAssertion(format!("Expected [... :db/valueType :db.type/*] but got [... :db/valueType {:?}]", value))) + for (ref symbolic_ident, ref symbolic_attr, ref value) in assertions.into_iter() { + let ident: i64 = *ident_map.get(symbolic_ident).ok_or(ErrorKind::UnrecognizedIdent(symbolic_ident.clone()))?; + let attr: i64 = *ident_map.get(symbolic_attr).ok_or(ErrorKind::UnrecognizedIdent(symbolic_attr.clone()))?; + let attributes = schema_map.entry(ident).or_insert(Attribute::default()); + + // TODO: improve error messages throughout. + match attr { + entids::DB_VALUE_TYPE => { + match *value { + TypedValue::Ref(entids::DB_TYPE_REF) => { attributes.value_type = ValueType::Ref; }, + TypedValue::Ref(entids::DB_TYPE_BOOLEAN) => { attributes.value_type = ValueType::Boolean; }, + TypedValue::Ref(entids::DB_TYPE_LONG) => { attributes.value_type = ValueType::Long; }, + TypedValue::Ref(entids::DB_TYPE_STRING) => { attributes.value_type = ValueType::String; }, + TypedValue::Ref(entids::DB_TYPE_KEYWORD) => { attributes.value_type = ValueType::Keyword; }, + _ => bail!(ErrorKind::BadSchemaAssertion(format!("Expected [... :db/valueType :db.type/*] but got [... :db/valueType {:?}] for ident '{}' and attribute '{}'", value, ident, attr))) } }, - ":db/cardinality" => { - if *value == *values::DB_CARDINALITY_MANY { - attributes.multival = true; - } else if *value == *values::DB_CARDINALITY_ONE { - attributes.multival = false; - } else { - bail!(ErrorKind::BadSchemaAssertion(format!("Expected [... :db/cardinality :db.cardinality/many|:db.cardinality/one] but got [... :db/cardinality {:?}]", value))) + + entids::DB_CARDINALITY => { + match *value { + TypedValue::Ref(entids::DB_CARDINALITY_MANY) => { attributes.multival = true; }, + TypedValue::Ref(entids::DB_CARDINALITY_ONE) => { attributes.multival = false; }, + _ => bail!(ErrorKind::BadSchemaAssertion(format!("Expected [... :db/cardinality :db.cardinality/many|:db.cardinality/one] but got [... :db/cardinality {:?}]", value))) } }, - ":db/unique" => { - if *value == *values::DB_UNIQUE_VALUE { - attributes.unique_value = true; - } else if *value == *values::DB_UNIQUE_IDENTITY { - attributes.unique_value = true; - attributes.unique_identity = true; - } else { - bail!(ErrorKind::BadSchemaAssertion(format!("Expected [... :db/unique :db.unique/value|:db.unique/identity] but got [... :db/unique {:?}]", value))) + + entids::DB_UNIQUE => { + match *value { + TypedValue::Ref(entids::DB_UNIQUE_VALUE) => { attributes.unique_value = true; }, + TypedValue::Ref(entids::DB_UNIQUE_IDENTITY) => { + attributes.unique_value = true; + attributes.unique_identity = true; + }, + _ => bail!(ErrorKind::BadSchemaAssertion(format!("Expected [... :db/unique :db.unique/value|:db.unique/identity] but got [... :db/unique {:?}]", value))) } }, - ":db/index" => { - if *value == Value::Boolean(true) { - attributes.index = true; - } else if *value == Value::Boolean(false) { - attributes.index = false; - } else { - bail!(ErrorKind::BadSchemaAssertion(format!("Expected [... :db/index true|false] but got [... :db/index {:?}]", value))) + + entids::DB_INDEX => { + match *value { + TypedValue::Boolean(x) => { attributes.index = x }, + _ => bail!(ErrorKind::BadSchemaAssertion(format!("Expected [... :db/index true|false] but got [... :db/index {:?}]", value))) } }, - ":db/fulltext" => { - if *value == Value::Boolean(true) { - attributes.index = true; - attributes.fulltext = true; - } else if *value == Value::Boolean(false) { - attributes.fulltext = false; - } else { - bail!(ErrorKind::BadSchemaAssertion(format!("Expected [... :db/fulltext true|false] but got [... :db/fulltext {:?}]", value))) + + entids::DB_FULLTEXT => { + match *value { + TypedValue::Boolean(x) => { + attributes.fulltext = x; + if attributes.fulltext { + attributes.index = true; + } + }, + _ => bail!(ErrorKind::BadSchemaAssertion(format!("Expected [... :db/fulltext true|false] but got [... :db/fulltext {:?}]", value))) } }, - ":db/isComponent" => { - if *value == Value::Boolean(true) { - attributes.component = true; - } else if *value == Value::Boolean(false) { - attributes.component = false; - } else { - bail!(ErrorKind::BadSchemaAssertion(format!("Expected [... :db/isComponent true|false] but got [... :db/isComponent {:?}]", value))) + + entids::DB_IS_COMPONENT => { + match *value { + TypedValue::Boolean(x) => { attributes.component = x }, + _ => bail!(ErrorKind::BadSchemaAssertion(format!("Expected [... :db/isComponent true|false] but got [... :db/isComponent {:?}]", value))) } }, - ":db/doc" => { + + entids::DB_DOC => { // Nothing for now. }, - ":db/ident" => { + + entids::DB_IDENT => { // Nothing for now. }, - ":db.install/attribute" => { + + entids::DB_INSTALL_ATTRIBUTE => { // Nothing for now. }, + _ => { bail!(ErrorKind::BadSchemaAssertion(format!("Do not recognize attribute '{}' for ident '{}'", attr, ident))) } diff --git a/db/src/types.rs b/db/src/types.rs index b67c6210e..dd118e15b 100644 --- a/db/src/types.rs +++ b/db/src/types.rs @@ -12,6 +12,8 @@ use std::collections::{BTreeMap}; +use ordered_float::{OrderedFloat}; + /// Core types defining a Mentat knowledge base. /// /// At its core, Mentat maintains a set of assertions of the form entity-attribute-value (EAV). The @@ -47,19 +49,6 @@ use std::collections::{BTreeMap}; /// contiguous. That is, it is possible for a specific entid to have never been present in the /// system, even though its predecessor and successor are present. -// #[derive(Debug)] -// pub enum Error { -// RusqliteError(rusqlite::Error), -// BadSchemaAssertion(String), -// BadBootstrapDefinition(String), -// } - -// impl From for Error { -// fn from(e: rusqlite::Error) -> Error { -// Error::RusqliteError(e) -// } -// } - /// Represents one entid in the entid space. /// /// Per https://www.sqlite.org/datatype3.html (see also http://stackoverflow.com/a/8499544), SQLite @@ -77,38 +66,31 @@ pub enum ValueType { Long, Double, String, - UUID, - URI, Keyword, } -impl ValueType { - pub fn value_type_tag(&self) -> i32 { - match *self { - ValueType::Ref => 0, - ValueType::Boolean => 1, - ValueType::Instant => 4, - ValueType::Long => 5, // SQLite distinguishes integral from decimal types, allowing long and double to share a tag. - ValueType::Double => 5, // SQLite distinguishes integral from decimal types, allowing long and double to share a tag. - ValueType::String => 10, - ValueType::UUID => 11, - ValueType::URI => 12, - ValueType::Keyword => 13, - } - } +/// Represents a Mentat value in a particular value set. +// TODO: expand to include :db.type/{instant,url,uuid}. +#[derive(Clone,Debug,Eq,Hash,Ord,PartialOrd,PartialEq)] +pub enum TypedValue { + Ref(Entid), + Boolean(bool), + Long(i64), + Double(OrderedFloat), + // TODO: &str throughout? + String(String), + Keyword(String), +} - pub fn from_value_type_tag(value_type_tag: &i32) -> Option { - match *value_type_tag { - 0 => Some(ValueType::Ref), - 1 => Some(ValueType::Boolean), - 4 => Some(ValueType::Instant), - 5 => Some(ValueType::Long), // SQLite distinguishes integral from decimal types, allowing long and double to share a tag. - // 5 => Some(ValueType::Double), - 10 => Some(ValueType::String), - 11 => Some(ValueType::UUID), - 12 => Some(ValueType::URI), - 13 => Some(ValueType::Keyword), - _ => None +impl TypedValue { + pub fn value_type(&self) -> ValueType { + match self { + &TypedValue::Ref(_) => ValueType::Ref, + &TypedValue::Boolean(_) => ValueType::Boolean, + &TypedValue::Long(_) => ValueType::Long, + &TypedValue::Double(_) => ValueType::Double, + &TypedValue::String(_) => ValueType::String, + &TypedValue::Keyword(_) => ValueType::Keyword, } } } From 67b6f9163d28a6c7a56590842512a3cbec81dbf5 Mon Sep 17 00:00:00 2001 From: Nick Alexander Date: Wed, 25 Jan 2017 13:17:41 -0800 Subject: [PATCH 11/14] Review comment: Always uses bundled SQLite in rusqlite. This avoids (runtime) failures in Travis CI due to old SQLite versions. See https://github.com/jgallagher/rusqlite/commit/432966ac77e3cb27c4cd67d073788498336988a2. --- Cargo.toml | 6 +++++- db/Cargo.toml | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 8d2f557a6..dc3572ba8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,7 +6,11 @@ authors = ["Richard Newman ", "Nicholas Alexander Date: Wed, 25 Jan 2017 13:27:12 -0800 Subject: [PATCH 12/14] Review comment: Move semantics in `from_sql_value_pair`. --- db/src/db.rs | 18 +++++++++--------- db/src/debug.rs | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/db/src/db.rs b/db/src/db.rs index 5d3ba8856..734229d38 100644 --- a/db/src/db.rs +++ b/db/src/db.rs @@ -285,17 +285,17 @@ pub fn ensure_current_version(conn: &mut rusqlite::Connection) -> Result { impl TypedValue { /// Given a SQLite `value` and a `value_type_tag`, return the corresponding `TypedValue`. - pub fn from_sql_value_pair(value: &rusqlite::types::Value, value_type_tag: &i32) -> Result { + pub fn from_sql_value_pair(value: rusqlite::types::Value, value_type_tag: &i32) -> Result { match (*value_type_tag, value) { - (0, &rusqlite::types::Value::Integer(ref x)) => Ok(TypedValue::Ref(*x)), - (1, &rusqlite::types::Value::Integer(ref x)) => Ok(TypedValue::Boolean(0 != *x)), + (0, rusqlite::types::Value::Integer(x)) => Ok(TypedValue::Ref(x)), + (1, rusqlite::types::Value::Integer(x)) => Ok(TypedValue::Boolean(0 != x)), // SQLite distinguishes integral from decimal types, allowing long and double to // share a tag. - (5, &rusqlite::types::Value::Integer(ref x)) => Ok(TypedValue::Long(*x)), - (5, &rusqlite::types::Value::Real(ref x)) => Ok(TypedValue::Double((*x).into())), - (10, &rusqlite::types::Value::Text(ref x)) => Ok(TypedValue::String(x.clone())), - (13, &rusqlite::types::Value::Text(ref x)) => Ok(TypedValue::Keyword(x.clone())), - (_, value) => bail!(ErrorKind::BadSQLValuePair(value.clone(), *value_type_tag)), + (5, rusqlite::types::Value::Integer(x)) => Ok(TypedValue::Long(x)), + (5, rusqlite::types::Value::Real(x)) => Ok(TypedValue::Double(x.into())), + (10, rusqlite::types::Value::Text(x)) => Ok(TypedValue::String(x)), + (13, rusqlite::types::Value::Text(x)) => Ok(TypedValue::Keyword(x)), + (_, value) => bail!(ErrorKind::BadSQLValuePair(value, *value_type_tag)), } } @@ -373,7 +373,7 @@ pub fn read_schema(conn: &rusqlite::Connection, ident_map: &IdentMap) -> Result< let symbolic_attr: String = row.get_checked(1)?; let v: rusqlite::types::Value = row.get_checked(2)?; let value_type_tag: i32 = row.get_checked(3)?; - let typed_value = TypedValue::from_sql_value_pair(&v, &value_type_tag)?; + let typed_value = TypedValue::from_sql_value_pair(v, &value_type_tag)?; Ok((symbolic_ident, symbolic_attr, typed_value)) })?.collect(); diff --git a/db/src/debug.rs b/db/src/debug.rs index 20c1be25f..79bfc659a 100644 --- a/db/src/debug.rs +++ b/db/src/debug.rs @@ -52,7 +52,7 @@ pub fn datoms_after(conn: &rusqlite::Connection, db: &DB, tx: &i32) -> Result Date: Wed, 25 Jan 2017 13:28:57 -0800 Subject: [PATCH 13/14] Review comment: DB_EXCISE_BEFORE_T instead of ...BEFORET (no underscore). --- db/src/bootstrap.rs | 2 +- db/src/entids.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/db/src/bootstrap.rs b/db/src/bootstrap.rs index 2286b7739..1891b35e3 100644 --- a/db/src/bootstrap.rs +++ b/db/src/bootstrap.rs @@ -41,7 +41,7 @@ lazy_static! { (":db.part/tx", entids::DB_PART_TX), (":db/excise", entids::DB_EXCISE), (":db.excise/attrs", entids::DB_EXCISE_ATTRS), - (":db.excise/beforeT", entids::DB_EXCISE_BEFORET), + (":db.excise/beforeT", entids::DB_EXCISE_BEFORE_T), (":db.excise/before", entids::DB_EXCISE_BEFORE), (":db.alter/attribute", entids::DB_ALTER_ATTRIBUTE), (":db.type/ref", entids::DB_TYPE_REF), diff --git a/db/src/entids.rs b/db/src/entids.rs index 3c508e598..f03b71b6a 100644 --- a/db/src/entids.rs +++ b/db/src/entids.rs @@ -36,7 +36,7 @@ pub const DB_PART_USER: Entid = 16; pub const DB_PART_TX: Entid = 17; pub const DB_EXCISE: Entid = 18; pub const DB_EXCISE_ATTRS: Entid = 19; -pub const DB_EXCISE_BEFORET: Entid = 20; +pub const DB_EXCISE_BEFORE_T: Entid = 20; pub const DB_EXCISE_BEFORE: Entid = 21; pub const DB_ALTER_ATTRIBUTE: Entid = 22; pub const DB_TYPE_REF: Entid = 23; From 1fe9041664e4e58ba066e0bb1b9f00068202df2d Mon Sep 17 00:00:00 2001 From: Nick Alexander Date: Wed, 25 Jan 2017 16:11:48 -0800 Subject: [PATCH 14/14] Review comment: Move overview notes to the Wiki. --- db/src/types.rs | 33 --------------------------------- 1 file changed, 33 deletions(-) diff --git a/db/src/types.rs b/db/src/types.rs index dd118e15b..ea8f66e7c 100644 --- a/db/src/types.rs +++ b/db/src/types.rs @@ -15,39 +15,6 @@ use std::collections::{BTreeMap}; use ordered_float::{OrderedFloat}; /// Core types defining a Mentat knowledge base. -/// -/// At its core, Mentat maintains a set of assertions of the form entity-attribute-value (EAV). The -/// assertions conform to a schema whereby the given attribute constrains the associated value/set -/// of associated values. -/// -/// ## Assertions -/// -/// Mentat assertions are represented as rows in the `datoms` SQLite table, and each Mentat row -/// representing an assertion is with a numeric representation of :db/valueType. -/// -/// The tag is used to limit queries, and therefore is placed carefully in the relevant indices to -/// allow searching numeric longs and doubles quickly. The tag is also used to convert SQLite -/// values to the correct Mentat value type on query egress. -/// -/// ## Entities and entids -/// -/// A Mentat entity is represented by a *positive* integer. (This agrees with Datomic.) We call -/// such a positive integer an *entid*. -/// -/// ## Partitions -/// -/// Datomic partitions the entid space in order to separate core knowledge base entities required -/// for the healthy function of the system from user-defined entities. Datomic also partitions in -/// order to ensure that certain index walks of related entities are efficient. Mentat follows -/// suit, partitioning into the following partitions: -/// * `:db.part/db`, for core knowledge base entities; -/// * `:db.part/user`, for user-defined entities; -/// * `:db.part/tx`, for transaction entities. -/// You almost certainly want to add new entities in the `:db.part/user` partition. -/// -/// The entid sequence in a given partition is monotonically increasing, although not necessarily -/// contiguous. That is, it is possible for a specific entid to have never been present in the -/// system, even though its predecessor and successor are present. /// Represents one entid in the entid space. ///