diff --git a/.github/workflows/service_test_mysql.yml b/.github/workflows/service_test_mysql.yml new file mode 100644 index 000000000000..a14abf85adcb --- /dev/null +++ b/.github/workflows/service_test_mysql.yml @@ -0,0 +1,65 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Service Test Mysql + +on: + push: + branches: + - main + pull_request: + branches: + - main + paths: + - "core/src/**" + - "core/tests/**" + - "!core/src/docs/**" + - "!core/src/services/**" + - "core/src/services/mysql/**" + - ".github/workflows/service_test_mysql.yml" + - "fixtures/mysql/**" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} + cancel-in-progress: true + +jobs: + mysql: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - name: Setup Mysql Server + shell: bash + working-directory: fixtures/mysql + run: docker-compose -f docker-compose.yml up -d + + - name: Setup Rust toolchain + uses: ./.github/actions/setup + with: + need-nextest: true + + - name: Test + shell: bash + working-directory: core + run: cargo nextest run mysql --features services-mysql + env: + OPENDAL_MYSQL_TEST: on + OPENDAL_MYSQL_CONNECTION_STRING: mysql://root:password@localhost:3306/testdb + OPENDAL_MYSQL_TABLE: data + OPENDAL_MYSQL_KEY_FIELD: key + OPENDAL_MYSQL_VALUE_FIELD: data diff --git a/Cargo.lock b/Cargo.lock index 10e8741c7006..30fa76d24f87 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -156,6 +156,12 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6" +[[package]] +name = "arrayvec" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" + [[package]] name = "assert-json-diff" version = "2.0.2" @@ -572,6 +578,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "bigdecimal" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6773ddc0eafc0e509fb60e48dff7f450f8e674a0686ae8605e8d9901bd5eefa" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + [[package]] name = "bincode" version = "1.3.3" @@ -636,6 +653,18 @@ version = "2.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "630be753d4e58660abd17930c71b647fe46c27ea6b63cc59e1e3851406972e42" +[[package]] +name = "bitvec" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + [[package]] name = "block-buffer" version = "0.10.4" @@ -660,6 +689,51 @@ dependencies = [ "log", ] +[[package]] +name = "borsh" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4114279215a005bc675e386011e594e1d9b800918cea18fcadadcce864a2046b" +dependencies = [ + "borsh-derive", + "hashbrown 0.13.2", +] + +[[package]] +name = "borsh-derive" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0754613691538d51f329cce9af41d7b7ca150bc973056f1156611489475f54f7" +dependencies = [ + "borsh-derive-internal", + "borsh-schema-derive-internal", + "proc-macro-crate 0.1.5", + "proc-macro2", + "syn 1.0.109", +] + +[[package]] +name = "borsh-derive-internal" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afb438156919598d2c7bad7e1c0adf3d26ed3840dbc010db1a882a65583ca2fb" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "borsh-schema-derive-internal" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "634205cc43f74a1b9046ef87c4540ebda95696ec0f315024860cad7c5b0f5ccd" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "bstr" version = "0.2.17" @@ -687,6 +761,28 @@ version = "3.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" +[[package]] +name = "bytecheck" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b6372023ac861f6e6dc89c8344a8f398fb42aaba2b5dbc649ca0c0e9dbcb627" +dependencies = [ + "bytecheck_derive", + "ptr_meta", + "simdutf8", +] + +[[package]] +name = "bytecheck_derive" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7ec4c6f261935ad534c0c22dbef2201b45918860eb1c574b972bd213a76af61" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "bytecount" version = "0.6.3" @@ -968,6 +1064,15 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2da6da31387c7e4ef160ffab6d5e7f00c42626fe39aea70a7b0f1773f7dd6c1b" +[[package]] +name = "cmake" +version = "0.1.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31c789563b815f77f4250caee12365734369f942439b7defd71e18a48197130" +dependencies = [ + "cc", +] + [[package]] name = "coarsetime" version = "0.1.23" @@ -1166,6 +1271,20 @@ dependencies = [ "itertools", ] +[[package]] +name = "crossbeam" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2801af0d36612ae591caa9568261fddce32ce6e08a7275ea334a06a4ad021a2c" +dependencies = [ + "cfg-if", + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-epoch", + "crossbeam-queue", + "crossbeam-utils", +] + [[package]] name = "crossbeam-channel" version = "0.5.8" @@ -1200,6 +1319,16 @@ dependencies = [ "scopeguard", ] +[[package]] +name = "crossbeam-queue" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1cfb3ea8a53f37c40dea2c7bedcbd88bdfae54f5e2175d6ecaff1c988353add" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.16" @@ -1301,8 +1430,18 @@ version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850" dependencies = [ - "darling_core", - "darling_macro", + "darling_core 0.14.4", + "darling_macro 0.14.4", +] + +[[package]] +name = "darling" +version = "0.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0209d94da627ab5605dcccf08bb18afa5009cfbef48d8a8b7d7bdbc79be25c5e" +dependencies = [ + "darling_core 0.20.3", + "darling_macro 0.20.3", ] [[package]] @@ -1319,17 +1458,42 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "darling_core" +version = "0.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "177e3443818124b357d8e76f53be906d60937f0d3a90773a664fa63fa253e621" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.23", +] + [[package]] name = "darling_macro" version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" dependencies = [ - "darling_core", + "darling_core 0.14.4", "quote", "syn 1.0.109", ] +[[package]] +name = "darling_macro" +version = "0.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5" +dependencies = [ + "darling_core 0.20.3", + "quote", + "syn 2.0.23", +] + [[package]] name = "dashmap" version = "5.4.0" @@ -1365,7 +1529,7 @@ dependencies = [ "lazy_static", "libc", "log", - "lru", + "lru 0.11.0", "mime_guess", "parking_lot 0.12.1", "percent-encoding", @@ -1453,7 +1617,7 @@ version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c11bdc11a0c47bc7d37d582b5285da6849c96681023680b906673c5707af7b0f" dependencies = [ - "darling", + "darling 0.14.4", "proc-macro2", "quote", "syn 1.0.109", @@ -1703,7 +1867,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "88979a9b357bdd9b8ffea208133f0bd0fa7e4d4e18bed3242335ed7b27033cf2" dependencies = [ "anyhow", - "darling", + "darling 0.14.4", "ident_case", "lazy_static", "proc-macro2", @@ -1762,6 +1926,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b9429470923de8e8cbd4d2dc513535400b4b3fef0319fb5c4e1f520a7bef743" dependencies = [ "crc32fast", + "libz-sys", "miniz_oxide", ] @@ -1864,6 +2029,58 @@ dependencies = [ "bindgen 0.65.1", ] +[[package]] +name = "frunk" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11a351b59e12f97b4176ee78497dff72e4276fb1ceb13e19056aca7fa0206287" +dependencies = [ + "frunk_core", + "frunk_derives", + "frunk_proc_macros", +] + +[[package]] +name = "frunk_core" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af2469fab0bd07e64ccf0ad57a1438f63160c69b2e57f04a439653d68eb558d6" + +[[package]] +name = "frunk_derives" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fa992f1656e1707946bbba340ad244f0814009ef8c0118eb7b658395f19a2e" +dependencies = [ + "frunk_proc_macro_helpers", + "quote", + "syn 2.0.23", +] + +[[package]] +name = "frunk_proc_macro_helpers" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35b54add839292b743aeda6ebedbd8b11e93404f902c56223e51b9ec18a13d2c" +dependencies = [ + "frunk_core", + "proc-macro2", + "quote", + "syn 2.0.23", +] + +[[package]] +name = "frunk_proc_macros" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71b85a1d4a9a6b300b41c05e8e13ef2feca03e0334127f29eca9506a7fe13a93" +dependencies = [ + "frunk_core", + "frunk_proc_macro_helpers", + "quote", + "syn 2.0.23", +] + [[package]] name = "fs2" version = "0.4.3" @@ -1874,6 +2091,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + [[package]] name = "futures" version = "0.3.28" @@ -2110,12 +2333,18 @@ name = "hashbrown" version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +dependencies = [ + "ahash 0.7.6", +] [[package]] name = "hashbrown" version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" +dependencies = [ + "ahash 0.8.3", +] [[package]] name = "hashbrown" @@ -2580,7 +2809,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6971da4d9c3aa03c3d8f3ff0f4155b534aad021292003895a469716b2a230378" dependencies = [ "base64 0.21.2", - "pem", + "pem 1.1.1", "ring", "serde", "serde_json", @@ -2657,6 +2886,79 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" +[[package]] +name = "lexical" +version = "6.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7aefb36fd43fef7003334742cbf77b243fcd36418a1d1bdd480d613a67968f6" +dependencies = [ + "lexical-core", +] + +[[package]] +name = "lexical-core" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cde5de06e8d4c2faabc400238f9ae1c74d5412d03a7bd067645ccbc47070e46" +dependencies = [ + "lexical-parse-float", + "lexical-parse-integer", + "lexical-util", + "lexical-write-float", + "lexical-write-integer", +] + +[[package]] +name = "lexical-parse-float" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683b3a5ebd0130b8fb52ba0bdc718cc56815b6a097e28ae5a6997d0ad17dc05f" +dependencies = [ + "lexical-parse-integer", + "lexical-util", + "static_assertions", +] + +[[package]] +name = "lexical-parse-integer" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d0994485ed0c312f6d965766754ea177d07f9c00c9b82a5ee62ed5b47945ee9" +dependencies = [ + "lexical-util", + "static_assertions", +] + +[[package]] +name = "lexical-util" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5255b9ff16ff898710eb9eb63cb39248ea8a5bb036bea8085b1a767ff6c4e3fc" +dependencies = [ + "static_assertions", +] + +[[package]] +name = "lexical-write-float" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accabaa1c4581f05a3923d1b4cfd124c329352288b7b9da09e766b0668116862" +dependencies = [ + "lexical-util", + "lexical-write-integer", + "static_assertions", +] + +[[package]] +name = "lexical-write-integer" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1b6f3d1f4422866b68192d62f77bc5c700bee84f3069f2469d7bc8c77852446" +dependencies = [ + "lexical-util", + "static_assertions", +] + [[package]] name = "libc" version = "0.2.147" @@ -2791,6 +3093,15 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "lru" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "718e8fae447df0c7e1ba7f5189829e63fd536945c8988d61444c19039f16b670" +dependencies = [ + "hashbrown 0.13.2", +] + [[package]] name = "lru" version = "0.11.0" @@ -2864,7 +3175,7 @@ version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3d248e97b1a48826a12c3828d921e8548e714394bf17274dd0a93910dc946e1" dependencies = [ - "darling", + "darling 0.14.4", "proc-macro2", "quote", "syn 1.0.109", @@ -3101,6 +3412,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" dependencies = [ "libc", + "log", "wasi 0.11.0+wasi-snapshot-preview1", "windows-sys 0.48.0", ] @@ -3163,6 +3475,95 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" +[[package]] +name = "mysql-common-derive" +version = "0.30.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56b0d8a0db9bf6d2213e11f2c701cb91387b0614361625ab7b9743b41aa4938f" +dependencies = [ + "darling 0.20.3", + "heck", + "num-bigint", + "proc-macro-crate 1.3.1", + "proc-macro-error", + "proc-macro2", + "quote", + "syn 2.0.23", + "termcolor", + "thiserror", +] + +[[package]] +name = "mysql_async" +version = "0.32.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5272f59b5b1f93d65f7f826c1f025d6e410e89fb50a67e05aa20b35a55a8c0a" +dependencies = [ + "bytes", + "crossbeam", + "flate2", + "futures-core", + "futures-sink", + "futures-util", + "lazy_static", + "lru 0.10.1", + "mio", + "mysql_common", + "native-tls", + "once_cell", + "pem 2.0.1", + "percent-encoding", + "pin-project", + "priority-queue", + "serde", + "serde_json", + "socket2 0.5.3", + "thiserror", + "tokio", + "tokio-native-tls", + "tokio-util", + "twox-hash", + "url", +] + +[[package]] +name = "mysql_common" +version = "0.30.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57349d5a326b437989b6ee4dc8f2f34b0cc131202748414712a8e7d98952fc8c" +dependencies = [ + "base64 0.21.2", + "bigdecimal", + "bindgen 0.65.1", + "bitflags 2.3.3", + "bitvec", + "byteorder", + "bytes", + "cc", + "cmake", + "crc32fast", + "flate2", + "frunk", + "lazy_static", + "lexical", + "mysql-common-derive", + "num-bigint", + "num-traits", + "rand 0.8.5", + "regex", + "rust_decimal", + "saturating", + "serde", + "serde_json", + "sha1", + "sha2", + "smallvec", + "subprocess", + "thiserror", + "time 0.3.22", + "uuid", +] + [[package]] name = "naive-timer" version = "0.2.0" @@ -3574,6 +3975,7 @@ dependencies = [ "minitrace", "minitrace-jaeger", "moka", + "mysql_async", "once_cell", "openssh", "openssh-sftp-client", @@ -4155,6 +4557,16 @@ dependencies = [ "base64 0.13.1", ] +[[package]] +name = "pem" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b13fe415cdf3c8e44518e18a7c95a13431d9bdf6d15367d82b23c377fdd441a" +dependencies = [ + "base64 0.21.2", + "serde", +] + [[package]] name = "pem-rfc7468" version = "0.7.0" @@ -4430,6 +4842,35 @@ dependencies = [ "syn 2.0.23", ] +[[package]] +name = "priority-queue" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fff39edfcaec0d64e8d0da38564fad195d2d51b680940295fcc307366e101e61" +dependencies = [ + "autocfg", + "indexmap 1.9.3", +] + +[[package]] +name = "proc-macro-crate" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d6ea3c4595b96363c13943497db34af4460fb474a95c43f4446ad341b8c9785" +dependencies = [ + "toml 0.5.11", +] + +[[package]] +name = "proc-macro-crate" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f4c021e1093a56626774e81216a4ce732a735e5bad4868a03f3ed65ca0c3919" +dependencies = [ + "once_cell", + "toml_edit", +] + [[package]] name = "proc-macro-error" version = "1.0.4" @@ -4583,6 +5024,26 @@ version = "2.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94" +[[package]] +name = "ptr_meta" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0738ccf7ea06b608c10564b31debd4f5bc5e197fc8bfe088f68ae5ce81e7a4f1" +dependencies = [ + "ptr_meta_derive", +] + +[[package]] +name = "ptr_meta_derive" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16b845dbfca988fa33db069c0e230574d15a3088f147a87b64c7589eb662c9ac" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "pulldown-cmark" version = "0.9.3" @@ -4724,6 +5185,12 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + [[package]] name = "rand" version = "0.7.3" @@ -4990,6 +5457,15 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" +[[package]] +name = "rend" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "581008d2099240d37fb08d77ad713bcaec2c4d89d50b5b21a8bb1996bbab68ab" +dependencies = [ + "bytecheck", +] + [[package]] name = "reqsign" version = "0.14.1" @@ -5097,6 +5573,34 @@ dependencies = [ "winapi", ] +[[package]] +name = "rkyv" +version = "0.7.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0200c8230b013893c0b2d6213d6ec64ed2b9be2e0e016682b7224ff82cff5c58" +dependencies = [ + "bitvec", + "bytecheck", + "hashbrown 0.12.3", + "ptr_meta", + "rend", + "rkyv_derive", + "seahash", + "tinyvec", + "uuid", +] + +[[package]] +name = "rkyv_derive" +version = "0.7.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2e06b915b5c230a17d7a736d1e2e63ee753c256a8614ef3f5147b13a4f5541d" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "rocksdb" version = "0.21.0" @@ -5139,6 +5643,22 @@ dependencies = [ "ordered-multimap", ] +[[package]] +name = "rust_decimal" +version = "1.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4c4216490d5a413bc6d10fa4742bd7d4955941d062c0ef873141d6b0e7b30fd" +dependencies = [ + "arrayvec", + "borsh", + "bytes", + "num-traits", + "rand 0.8.5", + "rkyv", + "serde", + "serde_json", +] + [[package]] name = "rustc-demangle" version = "0.1.23" @@ -5288,6 +5808,12 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "saturating" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ece8e78b2f38ec51c51f5d475df0a7187ba5111b2a28bdc761ee05b075d40a71" + [[package]] name = "schannel" version = "0.1.22" @@ -5344,6 +5870,12 @@ dependencies = [ "untrusted", ] +[[package]] +name = "seahash" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" + [[package]] name = "security-framework" version = "2.9.1" @@ -5543,6 +6075,12 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "simdutf8" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f27f6278552951f1f2b8cf9da965d10969b2efdea95a6ec47987ab46edfe263a" + [[package]] name = "simple_asn1" version = "0.6.2" @@ -5745,6 +6283,16 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +[[package]] +name = "subprocess" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c2e86926081dda636c546d8c5e641661049d7562a68f5488be4a1f7f66f6086" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "subtle" version = "2.5.0" @@ -5800,6 +6348,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + [[package]] name = "target-lexicon" version = "0.12.8" @@ -6464,6 +7018,17 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fb1626d07cb5c1bb2cf17d94c0be4852e8a7c02b041acec9a8c5bdda99f9d580" +[[package]] +name = "twox-hash" +version = "1.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" +dependencies = [ + "cfg-if", + "rand 0.8.5", + "static_assertions", +] + [[package]] name = "typenum" version = "1.16.0" @@ -7030,6 +7595,15 @@ dependencies = [ "tokio", ] +[[package]] +name = "wyz" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] + [[package]] name = "xml-rs" version = "0.8.16" diff --git a/core/Cargo.toml b/core/Cargo.toml index e09d94c5116f..f37cacb5c42d 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -176,6 +176,7 @@ services-wasabi = [ ] services-webdav = [] services-webhdfs = [] +services-mysql = ["dep:mysql_async"] [lib] bench = false @@ -273,6 +274,8 @@ tokio = "1.27" tokio-postgres = { version = "0.7.8", optional = true } tracing = { version = "0.1", optional = true } uuid = { version = "1", features = ["serde", "v4"] } +mysql_async = { version = "0.32.2", optional = true } + [dev-dependencies] criterion = { version = "0.4", features = ["async", "async_tokio"] } @@ -295,4 +298,4 @@ tracing-subscriber = { version = "0.3", features = [ "env-filter", "tracing-log", ] } -wiremock = "0.5" +wiremock = "0.5" \ No newline at end of file diff --git a/core/src/services/mod.rs b/core/src/services/mod.rs index 496c29f5a4f9..d50c2b85354c 100644 --- a/core/src/services/mod.rs +++ b/core/src/services/mod.rs @@ -213,3 +213,8 @@ pub use self::postgresql::Postgresql; mod atomicserver; #[cfg(feature = "services-atomicserver")] pub use self::atomicserver::Atomicserver; + +#[cfg(feature = "services-mysql")] +mod mysql; +#[cfg(feature = "services-mysql")] +pub use self::mysql::Mysql; diff --git a/core/src/services/mysql/backend.rs b/core/src/services/mysql/backend.rs new file mode 100644 index 000000000000..d6d8c9966448 --- /dev/null +++ b/core/src/services/mysql/backend.rs @@ -0,0 +1,294 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::collections::HashMap; +use std::fmt::Debug; + +use async_trait::async_trait; +use mysql_async::prelude::*; +use mysql_async::Opts; +use mysql_async::Pool; + +use crate::raw::adapters::kv; +use crate::raw::*; +use crate::*; + +#[doc = include_str!("docs.md")] +#[derive(Default)] +pub struct MysqlBuilder { + connection_string: Option, + + table: Option, + key_field: Option, + value_field: Option, + root: Option, +} + +impl Debug for MysqlBuilder { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("MysqlBuilder") + .field("connection_string", &self.connection_string) + .field("table", &self.table) + .field("key_field", &self.key_field) + .field("value_field", &self.value_field) + .field("root", &self.root) + .finish() + } +} + +impl MysqlBuilder { + /// Set the connection_string of the mysql service. + /// + /// This connection string is used to connect to the mysql service. There are url based formats: + /// + /// ## Url + /// + /// This format resembles the url format of the mysql client. The format is: [scheme://][user[:[password]]@]host[:port][/schema][?attribute1=value1&attribute2=value2... + /// + /// - `mysql://user@localhost` + /// - `mysql://user:password@localhost` + /// - `mysql://user:password@localhost:3306` + /// - `mysql://user:password@localhost:3306/db` + /// + /// For more information, please refer to [mysql client](https://dev.mysql.com/doc/refman/8.0/en/connecting-using-uri-or-key-value-pairs.html) + /// + pub fn connection_string(&mut self, v: &str) -> &mut Self { + if !v.is_empty() { + self.connection_string = Some(v.to_string()); + } + self + } + + /// set the working directory, all operations will be performed under it. + /// + /// default: "/" + pub fn root(&mut self, root: &str) -> &mut Self { + if !root.is_empty() { + self.root = Some(root.to_string()); + } + self + } + + /// Set the table name of the mysql service to read/write. + pub fn table(&mut self, table: &str) -> &mut Self { + if !table.is_empty() { + self.table = Some(table.to_string()); + } + self + } + + /// Set the key field name of the mysql service to read/write. + /// + /// Default to `key` if not specified. + pub fn key_field(&mut self, key_field: &str) -> &mut Self { + if !key_field.is_empty() { + self.key_field = Some(key_field.to_string()); + } + self + } + + /// Set the value field name of the mysql service to read/write. + /// + /// Default to `value` if not specified. + pub fn value_field(&mut self, value_field: &str) -> &mut Self { + if !value_field.is_empty() { + self.value_field = Some(value_field.to_string()); + } + self + } +} + +impl Builder for MysqlBuilder { + const SCHEME: Scheme = Scheme::Mysql; + type Accessor = MySqlBackend; + + fn from_map(map: HashMap) -> Self { + let mut builder = MysqlBuilder::default(); + map.get("connection_string") + .map(|v| builder.connection_string(v)); + map.get("table").map(|v| builder.table(v)); + map.get("key_field").map(|v| builder.key_field(v)); + map.get("value_field").map(|v| builder.value_field(v)); + map.get("root").map(|v| builder.root(v)); + builder + } + + fn build(&mut self) -> Result { + let conn = match self.connection_string.clone() { + Some(v) => v, + None => { + return Err( + Error::new(ErrorKind::ConfigInvalid, "connection_string is empty") + .with_context("service", Scheme::Mysql), + ) + } + }; + + let config = Opts::from_url(&conn).map_err(|err| { + Error::new(ErrorKind::ConfigInvalid, "connection_string is invalid") + .with_context("service", Scheme::Mysql) + .set_source(err) + })?; + + let table = match self.table.clone() { + Some(v) => v, + None => { + return Err(Error::new(ErrorKind::ConfigInvalid, "table is empty") + .with_context("service", Scheme::Mysql)) + } + }; + let key_field = match self.key_field.clone() { + Some(v) => v, + None => "key".to_string(), + }; + let value_field = match self.value_field.clone() { + Some(v) => v, + None => "value".to_string(), + }; + let root = normalize_root( + self.root + .clone() + .unwrap_or_else(|| "/".to_string()) + .as_str(), + ); + let pool = Pool::new(config.clone()); + + Ok(MySqlBackend::new(Adapter { + connection_pool: pool, + config, + table, + key_field, + value_field, + }) + .with_root(&root)) + } +} + +/// Backend for mysql service +pub type MySqlBackend = kv::Backend; + +#[derive(Clone)] +pub struct Adapter { + connection_pool: Pool, + config: Opts, + + table: String, + key_field: String, + value_field: String, +} + +impl Debug for Adapter { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Adapter") + .field("connection_pool", &self.connection_pool) + .field("config", &self.config) + .field("table", &self.table) + .field("key_field", &self.key_field) + .field("value_field", &self.value_field) + .finish() + } +} + +#[async_trait] +impl kv::Adapter for Adapter { + fn metadata(&self) -> kv::Metadata { + kv::Metadata::new( + Scheme::Mysql, + &self.table, + Capability { + read: true, + write: true, + ..Default::default() + }, + ) + } + + async fn get(&self, path: &str) -> Result>> { + let query = format!( + "SELECT `{}` FROM `{}` WHERE `{}` = :path LIMIT 1", + self.value_field, self.table, self.key_field + ); + let mut conn = self.connection_pool.get_conn().await.map_err(|err| { + Error::new(ErrorKind::Unexpected, "connection failed").set_source(err) + })?; + let statement = conn.prep(query).await.map_err(|err| { + Error::new(ErrorKind::Unexpected, "prepare statement failed").set_source(err) + })?; + let result: Option> = conn + .exec_first( + statement, + params! { + "path" => path, + }, + ) + .await + .map_err(|err| Error::new(ErrorKind::Unexpected, "delete failed").set_source(err))?; + match result { + Some(v) => Ok(Some(v)), + None => Ok(None), + } + } + + async fn set(&self, path: &str, value: &[u8]) -> Result<()> { + let query = format!( + "INSERT INTO `{}` (`{}`, `{}`) + VALUES (:path, :value) + ON DUPLICATE KEY UPDATE `{}` = VALUES({})", + self.table, self.key_field, self.value_field, self.value_field, self.value_field + ); + let mut conn = self.connection_pool.get_conn().await.map_err(|err| { + Error::new(ErrorKind::Unexpected, "connection failed").set_source(err) + })?; + let statement = conn.prep(query).await.map_err(|err| { + Error::new(ErrorKind::Unexpected, "prepare statement failed").set_source(err) + })?; + + conn.exec_drop( + statement, + params! { + "path" => path, + "value" => value, + }, + ) + .await + .map_err(|err| Error::new(ErrorKind::Unexpected, "set failed").set_source(err))?; + Ok(()) + } + + async fn delete(&self, path: &str) -> Result<()> { + let query = format!( + "DELETE FROM `{}` WHERE `{}` = :path", + self.table, self.key_field + ); + let mut conn = self.connection_pool.get_conn().await.map_err(|err| { + Error::new(ErrorKind::Unexpected, "connection failed").set_source(err) + })?; + let statement = conn.prep(query).await.map_err(|err| { + Error::new(ErrorKind::Unexpected, "prepare statement failed").set_source(err) + })?; + + conn.exec_drop( + statement, + params! { + "path" => path, + }, + ) + .await + .map_err(|err| Error::new(ErrorKind::Unexpected, "delete failed").set_source(err))?; + Ok(()) + } +} diff --git a/core/src/services/mysql/docs.md b/core/src/services/mysql/docs.md new file mode 100644 index 000000000000..3dfbcff46e00 --- /dev/null +++ b/core/src/services/mysql/docs.md @@ -0,0 +1,48 @@ +## Capabilities + +This service can be used to: + +- [x] stat +- [x] read +- [x] write +- [x] create_dir +- [x] delete +- [ ] copy +- [ ] rename +- [ ] ~~list~~ +- [ ] scan +- [ ] ~~presign~~ +- [ ] blocking + +## Configuration + +- `root`: Set the working directory of `OpenDAL` +- `connection_string`: Set the connection string of postgres server +- `table`: Set the table of mysql +- `key_field`: Set the key field of mysql +- `value_field`: Set the value field of mysql + +## Example + +### Via Builder + +```rust +use anyhow::Result; +use opendal::services::MySql; +use opendal::Operator; + +#[tokio::main] +async fn main() -> Result<()> { + let mut builder = MySql::default(); + builder.root("/"); + builder.connection_string("mysql://you_username:your_password@127.0.0.1:5432/your_database"); + builder.table("your_table"); + // key field type in the table should be compatible with Rust's &str like text + builder.key_field("key"); + // value field type in the table should be compatible with Rust's Vec like bytea + builder.value_field("value"); + + let op = Operator::new(builder)?.finish(); + Ok(()) +} +``` diff --git a/core/src/services/mysql/mod.rs b/core/src/services/mysql/mod.rs new file mode 100644 index 000000000000..8ed0254a502a --- /dev/null +++ b/core/src/services/mysql/mod.rs @@ -0,0 +1,19 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +mod backend; +pub use backend::MysqlBuilder as Mysql; diff --git a/core/src/services/postgresql/docs.md b/core/src/services/postgresql/docs.md index 129e6c5e54bc..e3f044b830ae 100644 --- a/core/src/services/postgresql/docs.md +++ b/core/src/services/postgresql/docs.md @@ -45,4 +45,4 @@ async fn main() -> Result<()> { let op = Operator::new(builder)?.finish(); Ok(()) } -``` \ No newline at end of file +``` diff --git a/core/src/types/scheme.rs b/core/src/types/scheme.rs index 142d90d4fbe0..e2af80f64332 100644 --- a/core/src/types/scheme.rs +++ b/core/src/types/scheme.rs @@ -88,6 +88,8 @@ pub enum Scheme { Redis, /// [postgresql][crate::services::Postgresql]: Postgresql services Postgresql, + /// [mysql][crate::services::Mysql]: Mysql services + Mysql, /// [rocksdb][crate::services::Rocksdb]: RocksDB services Rocksdb, /// [s3][crate::services::S3]: AWS S3 alike services. @@ -167,6 +169,7 @@ impl FromStr for Scheme { "ipmfs" => Ok(Scheme::Ipmfs), "memcached" => Ok(Scheme::Memcached), "memory" => Ok(Scheme::Memory), + "mysql" => Ok(Scheme::Mysql), "mini_moka" => Ok(Scheme::MiniMoka), "moka" => Ok(Scheme::Moka), "obs" => Ok(Scheme::Obs), @@ -218,6 +221,7 @@ impl From for &'static str { Scheme::Onedrive => "onedrive", Scheme::Persy => "persy", Scheme::Postgresql => "postgresql", + Scheme::Mysql => "mysql", Scheme::Gdrive => "gdrive", Scheme::Dropbox => "dropbox", Scheme::Redis => "redis", diff --git a/core/tests/behavior/main.rs b/core/tests/behavior/main.rs index 188da6365d3c..d9fc670437ab 100644 --- a/core/tests/behavior/main.rs +++ b/core/tests/behavior/main.rs @@ -178,6 +178,8 @@ fn main() -> anyhow::Result<()> { tests.extend(behavior_test::()); #[cfg(feature = "services-tikv")] tests.extend(behavior_test::()); + #[cfg(feature = "services-mysql")] + tests.extend(behavior_test::()); // Don't init logging while building operator which may break cargo // nextest output diff --git a/fixtures/mysql/docker-compose.yml b/fixtures/mysql/docker-compose.yml new file mode 100644 index 000000000000..9649218094ab --- /dev/null +++ b/fixtures/mysql/docker-compose.yml @@ -0,0 +1,34 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +version: '3.8' + +services: + mysql-db: + image: mysql:8.1.0 + ports: + - 3306:3306 + command: --max_allowed_packet=1024M + environment: + - MYSQL_ROOT_PASSWORD=password + - MYSQL_DATABASE=testdb + volumes: + - ./init.sql:/docker-entrypoint-initdb.d/init.sql + healthcheck: + test: ["CMD", "mysqladmin" ,"ping", "-h", "localhost"] + timeout: 20s + retries: 10 \ No newline at end of file diff --git a/fixtures/mysql/init.sql b/fixtures/mysql/init.sql new file mode 100644 index 000000000000..6c12aad38886 --- /dev/null +++ b/fixtures/mysql/init.sql @@ -0,0 +1,23 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- + +CREATE TABLE IF NOT EXISTS `data` ( + `key` VARCHAR(255) PRIMARY KEY, + `data` LONGBLOB +); \ No newline at end of file