Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
96c3bf9
ARROW-13542: Created dataset writer
westonpace Aug 20, 2021
dac4e0d
ARROW-13542: Adapted to use async smart pointers and async task group…
westonpace Sep 4, 2021
e45ecf0
ARROW-13542: Rebased onto async utilities merged into master & addres…
westonpace Sep 10, 2021
c9d91ca
ARROW-13542: Fixed python bindings (defualt to use_async=True for wri…
westonpace Sep 10, 2021
5fcf0aa
ARROW-13542: Added R support by changing write_dataset to create an a…
westonpace Sep 10, 2021
96b689b
ARROW-13542: Fixed race condition in counting semaphore test. Decrea…
westonpace Sep 10, 2021
38f7960
ARROW-13542: Trying to fix more tests/build errors
westonpace Sep 11, 2021
32e94c1
ARROW-13542: Found a bug in dataset_writer_test::DatasetWriterTestFix…
westonpace Sep 14, 2021
339d548
ARROW-13542: Fixed a bug in the async_util custom deleter that could …
westonpace Sep 14, 2021
c01a3df
ARROW-13542: Updated auto-generated R documentation
westonpace Sep 14, 2021
20067bc
ARROW-13542: Attempt at switching ruby to use async scanner
westonpace Sep 14, 2021
ec49b38
ARROW-13542: Fixed c_glib test to use async scanner for dataset writes
westonpace Sep 14, 2021
cf9cb0f
ARROW-13542: Need to call use_async on the scanner_builder and not th…
westonpace Sep 14, 2021
20fadb5
ARROW-13542: Touched up Ruby API
westonpace Sep 14, 2021
0c83f28
ARROW-13542: Removing formatting-only changes
westonpace Sep 15, 2021
440589b
ARROW-13542: Reverted changes to testing
westonpace Sep 15, 2021
724d922
ARROW-13542: Added verification of written file content to dataset wr…
westonpace Sep 15, 2021
f21d6de
ARROW-13542: Added condition to skip empty batches
westonpace Sep 21, 2021
0ff44b7
ARROW-13542: Addressing comments from PR review
westonpace Sep 28, 2021
98979fe
ARROW-13542: Fixing new test that was added with rebase and was using…
westonpace Sep 28, 2021
e06bf53
Apply suggestions from code review
westonpace Sep 29, 2021
3d354df
Update cpp/src/arrow/dataset/dataset_writer.cc
westonpace Sep 29, 2021
6759e3d
ARROW-13542: Addressing feedback from code review
westonpace Sep 29, 2021
2ac84d4
ARROW-13542: Addressing feedback from PR review
westonpace Sep 29, 2021
8f0d17b
ARROW-13542: Undoing changes not related to the issue
westonpace Sep 30, 2021
f352036
ARROW-13542: Fixing up import order for lint
westonpace Sep 30, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 19 additions & 0 deletions c_glib/arrow-dataset-glib/scanner.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -244,6 +244,25 @@ gadataset_scanner_builder_new_record_batch_reader(
return gadataset_scanner_builder_new_raw(&arrow_scanner_builder);
}

/**
* gadataset_scanner_builder_use_async:
* @builder: A #GADatasetScannerBuilder.
* @use_async: Use the asynchronous scanner
* @error: (nullable): Return location for a #GError or %NULL.
*
* Returns: void
*
* Since: 6.0.0
*/
void
gadataset_scanner_builder_use_async(GADatasetScannerBuilder *builder, gboolean use_async,
GError **error)
{
auto arrow_builder = gadataset_scanner_builder_get_raw(builder);
auto use_async_result = arrow_builder->UseAsync(use_async);
garrow::check(error, use_async_result, "[scanner-builder][use_async]");
}

/**
* gadataset_scanner_builder_finish:
* @builder: A #GADatasetScannerBuilder.
Expand Down
6 changes: 6 additions & 0 deletions c_glib/arrow-dataset-glib/scanner.h
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,12 @@ GARROW_AVAILABLE_IN_6_0
GADatasetScannerBuilder *
gadataset_scanner_builder_new_record_batch_reader(
GArrowRecordBatchReader *reader);

GARROW_AVAILABLE_IN_6_0
void
gadataset_scanner_builder_use_async(
GADatasetScannerBuilder *builder, gboolean use_async, GError **error);

GARROW_AVAILABLE_IN_5_0
GADatasetScanner *
gadataset_scanner_builder_finish(GADatasetScannerBuilder *builder,
Expand Down
1 change: 1 addition & 0 deletions c_glib/test/dataset/test-file-system-dataset.rb
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ def test_read_write
count: build_int32_array([1, 10, 2, 3]))
table_reader = Arrow::TableBatchReader.new(table)
scanner_builder = ArrowDataset::ScannerBuilder.new(table_reader)
scanner_builder.use_async(true)
scanner = scanner_builder.finish
options = ArrowDataset::FileSystemDatasetWriteOptions.new
options.file_write_options = @format.default_write_options
Expand Down
1 change: 1 addition & 0 deletions cpp/src/arrow/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -201,6 +201,7 @@ set(ARROW_SRCS
util/bpacking.cc
util/cancel.cc
util/compression.cc
util/counting_semaphore.cc
util/cpu_info.cc
util/decimal.cc
util/delimiting.cc
Expand Down
2 changes: 2 additions & 0 deletions cpp/src/arrow/dataset/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ arrow_install_all_headers("arrow/dataset")

set(ARROW_DATASET_SRCS
dataset.cc
dataset_writer.cc
discovery.cc
file_base.cc
file_ipc.cc
Expand Down Expand Up @@ -110,6 +111,7 @@ function(ADD_ARROW_DATASET_TEST REL_TEST_NAME)
endfunction()

add_arrow_dataset_test(dataset_test)
add_arrow_dataset_test(dataset_writer_test)
add_arrow_dataset_test(discovery_test)
add_arrow_dataset_test(file_ipc_test)
add_arrow_dataset_test(file_test)
Expand Down
Loading