Skip to content

ARROW-9749: [C++][GLib][Python][R][Ruby][Dataset] Introduce FragmentScanOptions, consolidate ScanContext/ScanOptions #9686

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
lidavidm wants to merge 4 commits into from
Closed

ARROW-9749: [C++][GLib][Python][R][Ruby][Dataset] Introduce FragmentScanOptions, consolidate ScanContext/ScanOptions #9686

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
3f9508a
ARROW-9749: [C++][Dataset] Add ConvertOptions for CSV datasets
lidavidm Mar 12, 2021
8622429
ARROW-9749: [C++][Python][Dataset][R] Eliminate all fields from ScanC…
lidavidm Mar 12, 2021
149cb23
ARROW-9749: [C++][Python][Dataset] Remove ScanContext
lidavidm Mar 12, 2021
5897a97
ARROW-9749: [GLib][Ruby] Remove ScanContext uses
lidavidm Mar 12, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
221 changes: 21 additions & 200 deletions c_glib/arrow-dataset-glib/scanner.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,6 @@ G_BEGIN_DECLS
* @title: Scanner classes
* @include: arrow-dataset-glib/arrow-dataset-glib.h
*
* #GADScanContext is a class for a scan context.
*
* #GADScanOptions is a class for a set of scan options.
*
* #GADScanTask is an abstract class for a scan task.
Expand All @@ -49,131 +47,6 @@ G_BEGIN_DECLS
* Since: 1.0.0
*/

/* arrow::dataset::ScanContext */

typedef struct GADScanContextPrivate_ {
std::shared_ptr<arrow::dataset::ScanContext> scan_context;
} GADScanContextPrivate;

enum {
PROP_SCAN_CONTEXT = 1,
PROP_USE_THREADS,
};

G_DEFINE_TYPE_WITH_PRIVATE(GADScanContext,
gad_scan_context,
G_TYPE_OBJECT)

#define GAD_SCAN_CONTEXT_GET_PRIVATE(obj) \
static_cast<GADScanContextPrivate *>( \
gad_scan_context_get_instance_private( \
GAD_SCAN_CONTEXT(obj)))

static void
gad_scan_context_finalize(GObject *object)
{
auto priv = GAD_SCAN_CONTEXT_GET_PRIVATE(object);

priv->scan_context.~shared_ptr();

G_OBJECT_CLASS(gad_scan_context_parent_class)->finalize(object);
}

static void
gad_scan_context_set_property(GObject *object,
guint prop_id,
const GValue *value,
GParamSpec *pspec)
{
auto priv = GAD_SCAN_CONTEXT_GET_PRIVATE(object);

switch (prop_id) {
case PROP_SCAN_CONTEXT:
priv->scan_context =
*static_cast<std::shared_ptr<arrow::dataset::ScanContext> *>(g_value_get_pointer(value));
break;
case PROP_USE_THREADS:
priv->scan_context->use_threads = g_value_get_boolean(value);
break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
break;
}
}

static void
gad_scan_context_get_property(GObject *object,
guint prop_id,
GValue *value,
GParamSpec *pspec)
{
auto priv = GAD_SCAN_CONTEXT_GET_PRIVATE(object);

switch (prop_id) {
case PROP_USE_THREADS:
g_value_set_boolean(value, priv->scan_context->use_threads);
break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
break;
}
}

static void
gad_scan_context_init(GADScanContext *object)
{
auto priv = GAD_SCAN_CONTEXT_GET_PRIVATE(object);
new(&priv->scan_context) std::shared_ptr<arrow::dataset::ScanContext>;
}

static void
gad_scan_context_class_init(GADScanContextClass *klass)
{
auto gobject_class = G_OBJECT_CLASS(klass);

gobject_class->finalize = gad_scan_context_finalize;
gobject_class->set_property = gad_scan_context_set_property;
gobject_class->get_property = gad_scan_context_get_property;

auto scan_context = arrow::dataset::ScanContext();

GParamSpec *spec;
spec = g_param_spec_pointer("scan-context",
"ScanContext",
"The raw std::shared<arrow::dataset::ScanContext> *",
static_cast<GParamFlags>(G_PARAM_WRITABLE |
G_PARAM_CONSTRUCT_ONLY));
g_object_class_install_property(gobject_class, PROP_SCAN_CONTEXT, spec);

/**
* GADScanContext:use-threads:
*
* Indicate if the Scanner should make use of a ThreadPool.
*
* Since: 1.0.0
*/
spec = g_param_spec_boolean("use-threads",
"Use threads",
"Indicate if the Scanner should make use of a ThreadPool",
scan_context.use_threads,
static_cast<GParamFlags>(G_PARAM_READWRITE));
g_object_class_install_property(gobject_class, PROP_USE_THREADS, spec);
}

/**
* gad_scan_context_new:
*
* Returns: A newly created #GADScanContext.
*
* Since: 1.0.0
*/
GADScanContext *
gad_scan_context_new(void)
{
auto arrow_scan_context = std::make_shared<arrow::dataset::ScanContext>();
return gad_scan_context_new_raw(&arrow_scan_context);
}

/* arrow::dataset::ScanOptions */

typedef struct GADScanOptionsPrivate_ {
Expand All @@ -186,6 +59,7 @@ enum {
PROP_EVALUATOR,
PROP_PROJECTOR,
PROP_BATCH_SIZE,
PROP_USE_THREADS,
};

G_DEFINE_TYPE_WITH_PRIVATE(GADScanOptions,
Expand Down Expand Up @@ -223,6 +97,9 @@ gad_scan_options_set_property(GObject *object,
case PROP_BATCH_SIZE:
priv->scan_options->batch_size = g_value_get_int64(value);
break;
case PROP_USE_THREADS:
priv->scan_options->use_threads = g_value_get_boolean(value);
break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
break;
Expand All @@ -241,6 +118,9 @@ gad_scan_options_get_property(GObject *object,
case PROP_BATCH_SIZE:
g_value_set_int64(value, priv->scan_options->batch_size);
break;
case PROP_USE_THREADS:
g_value_set_boolean(value, priv->scan_options->use_threads);
break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
break;
Expand Down Expand Up @@ -294,6 +174,20 @@ gad_scan_options_class_init(GADScanOptionsClass *klass)
scan_options->batch_size,
static_cast<GParamFlags>(G_PARAM_READWRITE));
g_object_class_install_property(gobject_class, PROP_BATCH_SIZE, spec);

/**
* GADScanOptions:use-threads:
*
* Indicate if the Scanner should make use of a ThreadPool.
*
* Since: 4.0.0
*/
spec = g_param_spec_boolean("use-threads",
"Use threads",
"Indicate if the Scanner should make use of a ThreadPool",
scan_options->use_threads,
static_cast<GParamFlags>(G_PARAM_READWRITE));
g_object_class_install_property(gobject_class, PROP_USE_THREADS, spec);
}

/**
Expand Down Expand Up @@ -334,14 +228,12 @@ gad_scan_options_get_schema(GADScanOptions *scan_options)
typedef struct GADScanTaskPrivate_ {
std::shared_ptr<arrow::dataset::ScanTask> scan_task;
GADScanOptions *options;
GADScanContext *context;
GADFragment *fragment;
} GADScanTaskPrivate;

enum {
PROP_SCAN_TASK = 1,
PROP_OPTIONS,
PROP_CONTEXT,
PROP_FRAGMENT,
};

Expand All @@ -364,11 +256,6 @@ gad_scan_task_dispose(GObject *object)
priv->options = NULL;
}

if (priv->context) {
g_object_unref(priv->context);
priv->context = NULL;
}

if (priv->fragment) {
g_object_unref(priv->fragment);
priv->fragment = NULL;
Expand Down Expand Up @@ -403,9 +290,6 @@ gad_scan_task_set_property(GObject *object,
case PROP_OPTIONS:
priv->options = GAD_SCAN_OPTIONS(g_value_dup_object(value));
break;
case PROP_CONTEXT:
priv->context = GAD_SCAN_CONTEXT(g_value_dup_object(value));
break;
case PROP_FRAGMENT:
priv->fragment = GAD_FRAGMENT(g_value_dup_object(value));
break;
Expand All @@ -427,9 +311,6 @@ gad_scan_task_get_property(GObject *object,
case PROP_OPTIONS:
g_value_set_object(value, priv->options);
break;
case PROP_CONTEXT:
g_value_set_object(value, priv->context);
break;
case PROP_FRAGMENT:
g_value_set_object(value, priv->fragment);
break;
Expand Down Expand Up @@ -479,21 +360,6 @@ gad_scan_task_class_init(GADScanTaskClass *klass)
G_PARAM_CONSTRUCT_ONLY));
g_object_class_install_property(gobject_class, PROP_OPTIONS, spec);

/**
* GADScanTask:context:
*
* The context of the scan task.
*
* Since: 1.0.0
*/
spec = g_param_spec_object("context",
"Context",
"The context of the scan task",
GAD_TYPE_SCAN_CONTEXT,
static_cast<GParamFlags>(G_PARAM_READWRITE |
G_PARAM_CONSTRUCT_ONLY));
g_object_class_install_property(gobject_class, PROP_CONTEXT, spec);

/**
* GADScanTask:fragment:
*
Expand Down Expand Up @@ -531,27 +397,6 @@ gad_scan_task_get_options(GADScanTask *scan_task)
return gad_scan_options_new_raw(&arrow_options);
}

/**
* gad_scan_task_get_context:
* @scan_task: A #GADScanTask.
*
* Returns: (transfer full): A #GADScanContext.
*
* Since: 1.0.0
*/
GADScanContext *
gad_scan_task_get_context(GADScanTask *scan_task)
{
auto priv = GAD_SCAN_TASK_GET_PRIVATE(scan_task);
if (priv->context) {
g_object_ref(priv->context);
return priv->context;
}

auto arrow_context = priv->scan_task->context();
return gad_scan_context_new_raw(&arrow_context);
}

/**
* gad_scan_task_get_fragment:
* @scan_task: A #GADFragment.
Expand Down Expand Up @@ -618,7 +463,6 @@ gad_in_memory_scan_task_class_init(GADInMemoryScanTaskClass *klass)
* (element-type GArrowRecordBatch): The record batches of the table.
* @n_record_batches: The number of record batches.
* @options: A #GADScanOptions.
* @context: A #GADScanContext.
* @fragment: A #GADInMemoryFragment.
*
* Returns: A newly created #GADInMemoryScanTask.
Expand All @@ -629,7 +473,6 @@ GADInMemoryScanTask *
gad_in_memory_scan_task_new(GArrowRecordBatch **record_batches,
gsize n_record_batches,
GADScanOptions *options,
GADScanContext *context,
GADInMemoryFragment *fragment)
{
std::vector<std::shared_ptr<arrow::RecordBatch>> arrow_record_batches;
Expand All @@ -639,38 +482,18 @@ gad_in_memory_scan_task_new(GArrowRecordBatch **record_batches,
arrow_record_batches.push_back(arrow_record_batch);
}
auto arrow_options = gad_scan_options_get_raw(options);
auto arrow_context = gad_scan_context_get_raw(context);
auto arrow_fragment = gad_fragment_get_raw(GAD_FRAGMENT(fragment));
auto arrow_in_memory_scan_task =
std::make_shared<arrow::dataset::InMemoryScanTask>(arrow_record_batches,
arrow_options,
arrow_context,
arrow_fragment);
return gad_in_memory_scan_task_new_raw(&arrow_in_memory_scan_task,
options,
context,
fragment);
}

G_END_DECLS

GADScanContext *
gad_scan_context_new_raw(std::shared_ptr<arrow::dataset::ScanContext> *arrow_scan_context)
{
auto scan_context =
GAD_SCAN_CONTEXT(g_object_new(GAD_TYPE_SCAN_CONTEXT,
"scan-context", arrow_scan_context,
NULL));
return scan_context;
}

std::shared_ptr<arrow::dataset::ScanContext>
gad_scan_context_get_raw(GADScanContext *scan_context)
{
auto priv = GAD_SCAN_CONTEXT_GET_PRIVATE(scan_context);
return priv->scan_context;
}

GADScanOptions *
gad_scan_options_new_raw(std::shared_ptr<arrow::dataset::ScanOptions> *arrow_scan_options)
{
Expand All @@ -691,14 +514,12 @@ gad_scan_options_get_raw(GADScanOptions *scan_options)
GADInMemoryScanTask *
gad_in_memory_scan_task_new_raw(std::shared_ptr<arrow::dataset::InMemoryScanTask> *arrow_in_memory_scan_task,
GADScanOptions *options,
GADScanContext *context,
GADInMemoryFragment *fragment)
{
auto in_memory_scan_task =
GAD_IN_MEMORY_SCAN_TASK(g_object_new(GAD_TYPE_IN_MEMORY_SCAN_TASK,
"scan-task", arrow_in_memory_scan_task,
"options", options,
"context", context,
"fragment", fragment,
NULL));
return in_memory_scan_task;
Expand Down
19 changes: 0 additions & 19 deletions c_glib/arrow-dataset-glib/scanner.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,22 +25,6 @@

G_BEGIN_DECLS

/* arrow::dataset::ScanContext */

#define GAD_TYPE_SCAN_CONTEXT (gad_scan_context_get_type())
G_DECLARE_DERIVABLE_TYPE(GADScanContext,
gad_scan_context,
GAD,
SCAN_CONTEXT,
GObject)
struct _GADScanContextClass
{
GObjectClass parent_class;
};

GARROW_AVAILABLE_IN_1_0
GADScanContext *gad_scan_context_new(void);

/* arrow::dataset::ScanOptions */

#define GAD_TYPE_SCAN_OPTIONS (gad_scan_options_get_type())
Expand Down Expand Up @@ -75,8 +59,6 @@ struct _GADScanTaskClass

GARROW_AVAILABLE_IN_1_0
GADScanOptions *gad_scan_task_get_options(GADScanTask *scan_task);
GARROW_AVAILABLE_IN_1_0
GADScanContext *gad_scan_task_get_context(GADScanTask *scan_task);
GARROW_AVAILABLE_IN_4_0
GADFragment *gad_scan_task_get_fragment(GADScanTask *scan_task);
GARROW_AVAILABLE_IN_1_0
Expand All @@ -101,7 +83,6 @@ GADInMemoryScanTask *
gad_in_memory_scan_task_new(GArrowRecordBatch **record_batches,
gsize n_record_batches,
GADScanOptions *options,
GADScanContext *context,
GADInMemoryFragment *fragment);

G_END_DECLS
Loading