From da4954695314a96b6033a8616e3eccafa87ecfd6 Mon Sep 17 00:00:00 2001 From: "Heres, Daniel" Date: Sun, 14 Feb 2021 17:16:23 +0100 Subject: [PATCH 01/10] Move datafusion examples to own project --- rust/Cargo.toml | 1 + rust/datafusion-examples/Cargo.toml | 38 +++++++++++++++++++ .../examples/README.md | 0 .../examples/csv_sql.rs | 0 .../examples/dataframe.rs | 0 .../examples/dataframe_in_memory.rs | 0 .../examples/flight_client.rs | 0 .../examples/flight_server.rs | 0 .../examples/parquet_sql.rs | 0 .../examples/simple_udaf.rs | 0 .../examples/simple_udf.rs | 0 11 files changed, 39 insertions(+) create mode 100644 rust/datafusion-examples/Cargo.toml rename rust/{datafusion => datafusion-examples}/examples/README.md (100%) rename rust/{datafusion => datafusion-examples}/examples/csv_sql.rs (100%) rename rust/{datafusion => datafusion-examples}/examples/dataframe.rs (100%) rename rust/{datafusion => datafusion-examples}/examples/dataframe_in_memory.rs (100%) rename rust/{datafusion => datafusion-examples}/examples/flight_client.rs (100%) rename rust/{datafusion => datafusion-examples}/examples/flight_server.rs (100%) rename rust/{datafusion => datafusion-examples}/examples/parquet_sql.rs (100%) rename rust/{datafusion => datafusion-examples}/examples/simple_udaf.rs (100%) rename rust/{datafusion => datafusion-examples}/examples/simple_udf.rs (100%) diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 16e34de7f14..c29900429b2 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -22,6 +22,7 @@ members = [ "parquet_derive", "parquet_derive_test", "datafusion", + "datafusion-examples", "arrow-flight", "integration-testing", "benchmarks", diff --git a/rust/datafusion-examples/Cargo.toml b/rust/datafusion-examples/Cargo.toml new file mode 100644 index 00000000000..6a588077edd --- /dev/null +++ b/rust/datafusion-examples/Cargo.toml @@ -0,0 +1,38 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[package] +name = "datafusion-examples" +description = "DataFusion usage examples" +version = "4.0.0-SNAPSHOT" +homepage = "https://github.com/apache/arrow" +repository = "https://github.com/apache/arrow" +authors = ["Apache Arrow "] +license = "Apache-2.0" +keywords = [ "arrow", "query", "sql" ] +edition = "2018" + + +[dev-dependencies] +datafusion = { path = "../datafusion" } +arrow = { path = "../arrow" } +prost = "0.7" +arrow-flight = { path = "../arrow-flight", version = "4.0.0-SNAPSHOT" } +tonic = "0.4" +tokio = { version = "1.0", features = ["macros", "rt", "rt-multi-thread", "sync"] } +futures = "0.3" +num_cpus = "1.13.0" diff --git a/rust/datafusion/examples/README.md b/rust/datafusion-examples/examples/README.md similarity index 100% rename from rust/datafusion/examples/README.md rename to rust/datafusion-examples/examples/README.md diff --git a/rust/datafusion/examples/csv_sql.rs b/rust/datafusion-examples/examples/csv_sql.rs similarity index 100% rename from rust/datafusion/examples/csv_sql.rs rename to rust/datafusion-examples/examples/csv_sql.rs diff --git a/rust/datafusion/examples/dataframe.rs b/rust/datafusion-examples/examples/dataframe.rs similarity index 100% rename from rust/datafusion/examples/dataframe.rs rename to rust/datafusion-examples/examples/dataframe.rs diff --git a/rust/datafusion/examples/dataframe_in_memory.rs b/rust/datafusion-examples/examples/dataframe_in_memory.rs similarity index 100% rename from rust/datafusion/examples/dataframe_in_memory.rs rename to rust/datafusion-examples/examples/dataframe_in_memory.rs diff --git a/rust/datafusion/examples/flight_client.rs b/rust/datafusion-examples/examples/flight_client.rs similarity index 100% rename from rust/datafusion/examples/flight_client.rs rename to rust/datafusion-examples/examples/flight_client.rs diff --git a/rust/datafusion/examples/flight_server.rs b/rust/datafusion-examples/examples/flight_server.rs similarity index 100% rename from rust/datafusion/examples/flight_server.rs rename to rust/datafusion-examples/examples/flight_server.rs diff --git a/rust/datafusion/examples/parquet_sql.rs b/rust/datafusion-examples/examples/parquet_sql.rs similarity index 100% rename from rust/datafusion/examples/parquet_sql.rs rename to rust/datafusion-examples/examples/parquet_sql.rs diff --git a/rust/datafusion/examples/simple_udaf.rs b/rust/datafusion-examples/examples/simple_udaf.rs similarity index 100% rename from rust/datafusion/examples/simple_udaf.rs rename to rust/datafusion-examples/examples/simple_udaf.rs diff --git a/rust/datafusion/examples/simple_udf.rs b/rust/datafusion-examples/examples/simple_udf.rs similarity index 100% rename from rust/datafusion/examples/simple_udf.rs rename to rust/datafusion-examples/examples/simple_udf.rs From 73235ba7b6eda05b363db437ba9e075ea88cc6b5 Mon Sep 17 00:00:00 2001 From: "Heres, Daniel" Date: Sun, 14 Feb 2021 17:19:11 +0100 Subject: [PATCH 02/10] Remove dependencies --- rust/datafusion/Cargo.toml | 3 --- 1 file changed, 3 deletions(-) diff --git a/rust/datafusion/Cargo.toml b/rust/datafusion/Cargo.toml index 94e56a41e7d..3236a97124d 100644 --- a/rust/datafusion/Cargo.toml +++ b/rust/datafusion/Cargo.toml @@ -75,9 +75,6 @@ lazy_static = { version = "^1.4.0", optional = true } rand = "0.8" criterion = "0.3" tempfile = "3" -prost = "0.7" -arrow-flight = { path = "../arrow-flight", version = "4.0.0-SNAPSHOT" } -tonic = "0.4" doc-comment = "0.3" [[bench]] From 225a6844dded23dc8d1166780b4673ac49d64d90 Mon Sep 17 00:00:00 2001 From: "Heres, Daniel" Date: Sun, 14 Feb 2021 19:55:45 +0100 Subject: [PATCH 03/10] Fix CI --- .github/workflows/rust.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 90831341d6c..81dbfab56be 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -116,7 +116,7 @@ jobs: # run tests on all workspace members with default feature list cargo test # test datafusion examples - cd datafusion + cd datafusion-examples cargo test --no-default-features --features cli cargo run --example csv_sql cargo run --example parquet_sql From cebc63d9d7632cd20b6f0b87ced13ed6727cc2ba Mon Sep 17 00:00:00 2001 From: "Heres, Daniel" Date: Sun, 14 Feb 2021 20:11:31 +0100 Subject: [PATCH 04/10] Add to prepare-test --- dev/release/00-prepare-test.rb | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/dev/release/00-prepare-test.rb b/dev/release/00-prepare-test.rb index b686ecd97d1..2b6f373b00d 100644 --- a/dev/release/00-prepare-test.rb +++ b/dev/release/00-prepare-test.rb @@ -288,6 +288,13 @@ def test_version_pre_tag "+version = \"#{@release_version}\""], ], }, + { + path: "rust/datafusion-examples/Cargo.toml", + hunks: [ + ["-version = \"#{@snapshot_version}\"", + "+version = \"#{@release_version}\""], + ], + }, { path: "rust/datafusion/Cargo.toml", hunks: [ From f776ce9e4294f1bb20177290a1df1ad42bfd74ee Mon Sep 17 00:00:00 2001 From: "Heres, Daniel" Date: Sun, 14 Feb 2021 20:17:37 +0100 Subject: [PATCH 05/10] Move arrow-flight --- dev/release/00-prepare-test.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/release/00-prepare-test.rb b/dev/release/00-prepare-test.rb index 2b6f373b00d..1e17e3e6fea 100644 --- a/dev/release/00-prepare-test.rb +++ b/dev/release/00-prepare-test.rb @@ -293,6 +293,8 @@ def test_version_pre_tag hunks: [ ["-version = \"#{@snapshot_version}\"", "+version = \"#{@release_version}\""], + ["-arrow-flight = { path = \"../arrow-flight\", version = \"#{@snapshot_version}\" }", + "+arrow-flight = { path = \"../arrow-flight\", version = \"#{@release_version}\" }"] ], }, { @@ -304,8 +306,6 @@ def test_version_pre_tag "-parquet = { path = \"../parquet\", version = \"#{@snapshot_version}\", features = [\"arrow\"] }", "+arrow = { path = \"../arrow\", version = \"#{@release_version}\", features = [\"prettyprint\"] }", "+parquet = { path = \"../parquet\", version = \"#{@release_version}\", features = [\"arrow\"] }"], - ["-arrow-flight = { path = \"../arrow-flight\", version = \"#{@snapshot_version}\" }", - "+arrow-flight = { path = \"../arrow-flight\", version = \"#{@release_version}\" }"] ], }, { From ac2f2eecb0624c862a44c8a6a57ca7316d28b7b2 Mon Sep 17 00:00:00 2001 From: "Heres, Daniel" Date: Sun, 14 Feb 2021 20:26:58 +0100 Subject: [PATCH 06/10] Use latest version --- dev/release/00-prepare-test.rb | 2 -- rust/datafusion-examples/Cargo.toml | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/dev/release/00-prepare-test.rb b/dev/release/00-prepare-test.rb index 1e17e3e6fea..95e203c995b 100644 --- a/dev/release/00-prepare-test.rb +++ b/dev/release/00-prepare-test.rb @@ -293,8 +293,6 @@ def test_version_pre_tag hunks: [ ["-version = \"#{@snapshot_version}\"", "+version = \"#{@release_version}\""], - ["-arrow-flight = { path = \"../arrow-flight\", version = \"#{@snapshot_version}\" }", - "+arrow-flight = { path = \"../arrow-flight\", version = \"#{@release_version}\" }"] ], }, { diff --git a/rust/datafusion-examples/Cargo.toml b/rust/datafusion-examples/Cargo.toml index 6a588077edd..2bd424c4883 100644 --- a/rust/datafusion-examples/Cargo.toml +++ b/rust/datafusion-examples/Cargo.toml @@ -31,7 +31,7 @@ edition = "2018" datafusion = { path = "../datafusion" } arrow = { path = "../arrow" } prost = "0.7" -arrow-flight = { path = "../arrow-flight", version = "4.0.0-SNAPSHOT" } +arrow-flight = { path = "../arrow-flight" } tonic = "0.4" tokio = { version = "1.0", features = ["macros", "rt", "rt-multi-thread", "sync"] } futures = "0.3" From 4626305930f4c775c857d090473ab15e0af44a6a Mon Sep 17 00:00:00 2001 From: "Heres, Daniel" Date: Sun, 14 Feb 2021 20:45:18 +0100 Subject: [PATCH 07/10] Add to both tests --- dev/release/00-prepare-test.rb | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/dev/release/00-prepare-test.rb b/dev/release/00-prepare-test.rb index 95e203c995b..11425163f3c 100644 --- a/dev/release/00-prepare-test.rb +++ b/dev/release/00-prepare-test.rb @@ -540,6 +540,13 @@ def test_version_post_tag "+version = \"#{@next_snapshot_version}\""], ], }, + { + path: "rust/datafusion-examples/Cargo.toml", + hunks: [ + ["-version = \"#{@snapshot_version}\"", + "+version = \"#{@release_version}\""], + ], + }, { path: "rust/datafusion/Cargo.toml", hunks: [ @@ -549,8 +556,6 @@ def test_version_post_tag "-parquet = { path = \"../parquet\", version = \"#{@release_version}\", features = [\"arrow\"] }", "+arrow = { path = \"../arrow\", version = \"#{@next_snapshot_version}\", features = [\"prettyprint\"] }", "+parquet = { path = \"../parquet\", version = \"#{@next_snapshot_version}\", features = [\"arrow\"] }"], - ["-arrow-flight = { path = \"../arrow-flight\", version = \"#{@release_version}\" }", - "+arrow-flight = { path = \"../arrow-flight\", version = \"#{@next_snapshot_version}\" }"] ], }, { From 94dadb053be1cf6c04ccf8ae68fa252132a1fabb Mon Sep 17 00:00:00 2001 From: "Heres, Daniel" Date: Sun, 14 Feb 2021 20:57:55 +0100 Subject: [PATCH 08/10] Fix --- dev/release/00-prepare-test.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dev/release/00-prepare-test.rb b/dev/release/00-prepare-test.rb index 11425163f3c..53bd5e89bf2 100644 --- a/dev/release/00-prepare-test.rb +++ b/dev/release/00-prepare-test.rb @@ -543,9 +543,9 @@ def test_version_post_tag { path: "rust/datafusion-examples/Cargo.toml", hunks: [ - ["-version = \"#{@snapshot_version}\"", - "+version = \"#{@release_version}\""], - ], + ["-version = \"#{@release_version}\"", + "+version = \"#{@next_snapshot_version}\""], + ], }, { path: "rust/datafusion/Cargo.toml", From 835bc47ebcd4aa3330f855e816ed5ed4029b5d68 Mon Sep 17 00:00:00 2001 From: "Heres, Daniel" Date: Mon, 22 Mar 2021 18:23:36 +0100 Subject: [PATCH 09/10] Add publish=false --- rust/datafusion-examples/Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/rust/datafusion-examples/Cargo.toml b/rust/datafusion-examples/Cargo.toml index 2bd424c4883..c86e7ccbe3c 100644 --- a/rust/datafusion-examples/Cargo.toml +++ b/rust/datafusion-examples/Cargo.toml @@ -25,6 +25,7 @@ authors = ["Apache Arrow "] license = "Apache-2.0" keywords = [ "arrow", "query", "sql" ] edition = "2018" +publish = false [dev-dependencies] From 19884c3d3e4d97fde9d92e03b211f289f6c5c35c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20Heres?= Date: Tue, 23 Mar 2021 18:44:48 +0100 Subject: [PATCH 10/10] Update .github/workflows/rust.yml Co-authored-by: Andrew Lamb --- .github/workflows/rust.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 81dbfab56be..1dd220ade94 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -117,7 +117,7 @@ jobs: cargo test # test datafusion examples cd datafusion-examples - cargo test --no-default-features --features cli + cargo test --no-default-features cargo run --example csv_sql cargo run --example parquet_sql cd ..