From e186fd7f6ae651f73f943c2c7e848bd275e3ffcb Mon Sep 17 00:00:00 2001 From: Joel Lubinitsky Date: Thu, 12 Oct 2023 17:54:30 -0400 Subject: [PATCH 1/5] Update flight sql proto --- format/FlightSql.proto | 49 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/format/FlightSql.proto b/format/FlightSql.proto index 3c9a719f127..34d6dca256c 100644 --- a/format/FlightSql.proto +++ b/format/FlightSql.proto @@ -817,6 +817,17 @@ enum SqlInfo { * - true: if invoking user-defined or vendor functions using the stored procedure escape syntax is supported. */ SQL_STORED_FUNCTIONS_USING_CALL_SYNTAX_SUPPORTED = 576; + + /* + * Retrieves a boolean value indicating whether transactions are supported for bulk ingestion. If not, invoking + * the method commit in the context of a bulk ingestion is a noop, and the isolation level is + * `arrow.flight.protocol.sql.SqlTransactionIsolationLevel.TRANSACTION_NONE`. + * + * Returns: + * - false: if bulk ingestion transactions are unsupported; + * - true: if bulk ingestion transactions are supported. + */ + INGEST_TRANSACTIONS_SUPPORTED = 577; } // The level of support for Flight SQL transaction RPCs. @@ -1778,6 +1789,44 @@ message CommandPreparedStatementUpdate { bytes prepared_statement_handle = 1; } + /* + * Represents a bulk ingestion request. Used in the command member of FlightDescriptor + * for the the RPC call DoPut to cause the server load the contents of the stream's + * FlightData into the target destination. + */ +message CommandStatementIngest { + option (experimental) = true; + + // Describes the behavior for loading bulk data. + enum IngestMode { + // Ingestion behavior unspecified. + INGEST_MODE_UNSPECIFIED = 0; + // Create the target table. Fail if the target table already exists. + INGEST_MODE_CREATE = 1; + // Append to an existing target table. Fail if the target table does not exist. + INGEST_MODE_APPEND = 2; + // Drop the target table if it exists. Then follow INGEST_MODE_CREATE behavior. + INGEST_MODE_REPLACE = 3; + // Create the target table if it does not exist. Then follow INGEST_MODE_APPEND behavior. + INGEST_MODE_CREATE_APPEND = 4; + } + + // The ingestion behavior. + IngestMode mode = 1; + // The table to load data into. + string target_table = 2; + // The db_schema of the target_table to load data into. If unset, ... (TODO) + optional string target_schema = 3; + // The catalog of the target_table to load data into. If unset, ... (TODO) + optional string target_catalog = 4; + // Use a temporary table for target_table. + optional bool temporary = 5; + // Perform the ingestion as part of this transaction (if unset, the query is auto-committed). + optional bool transaction_id = 6; + // Backend-specific options. + map options = 1000; +} + /* * Returned from the RPC call DoPut when a CommandStatementUpdate * CommandPreparedStatementUpdate was in the request, containing From d962c22822ae680d83689235128687560278042a Mon Sep 17 00:00:00 2001 From: Joel Lubinitsky Date: Fri, 13 Oct 2023 05:15:36 -0400 Subject: [PATCH 2/5] Clarify schema, catalog behavior in comment --- format/FlightSql.proto | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/format/FlightSql.proto b/format/FlightSql.proto index 34d6dca256c..f46d5d4b30e 100644 --- a/format/FlightSql.proto +++ b/format/FlightSql.proto @@ -1815,9 +1815,9 @@ message CommandStatementIngest { IngestMode mode = 1; // The table to load data into. string target_table = 2; - // The db_schema of the target_table to load data into. If unset, ... (TODO) + // The db_schema of the target_table to load data into. If unset, a backend-specific default may be used. optional string target_schema = 3; - // The catalog of the target_table to load data into. If unset, ... (TODO) + // The catalog of the target_table to load data into. If unset, a backend-specific default may be used. optional string target_catalog = 4; // Use a temporary table for target_table. optional bool temporary = 5; From 05011662d8be214b536e0e2fdef1eab14ff38f7f Mon Sep 17 00:00:00 2001 From: Joel Lubinitsky Date: Thu, 19 Oct 2023 06:18:36 -0400 Subject: [PATCH 3/5] Remove target_ prefix --- format/FlightSql.proto | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/format/FlightSql.proto b/format/FlightSql.proto index f46d5d4b30e..78c7b234f2a 100644 --- a/format/FlightSql.proto +++ b/format/FlightSql.proto @@ -1814,12 +1814,12 @@ message CommandStatementIngest { // The ingestion behavior. IngestMode mode = 1; // The table to load data into. - string target_table = 2; - // The db_schema of the target_table to load data into. If unset, a backend-specific default may be used. - optional string target_schema = 3; - // The catalog of the target_table to load data into. If unset, a backend-specific default may be used. - optional string target_catalog = 4; - // Use a temporary table for target_table. + string table = 2; + // The db_schema of the destination table to load data into. If unset, a backend-specific default may be used. + optional string schema = 3; + // The catalog of the destination table to load data into. If unset, a backend-specific default may be used. + optional string catalog = 4; + // Use a temporary table. optional bool temporary = 5; // Perform the ingestion as part of this transaction (if unset, the query is auto-committed). optional bool transaction_id = 6; From 6e8291726415fa58826f8cba40537a1b9c44449c Mon Sep 17 00:00:00 2001 From: Joel Lubinitsky Date: Thu, 19 Oct 2023 06:29:54 -0400 Subject: [PATCH 4/5] Add comment explaining extensions to spec --- format/FlightSql.proto | 3 +++ 1 file changed, 3 insertions(+) diff --git a/format/FlightSql.proto b/format/FlightSql.proto index 78c7b234f2a..fd141b7e2fb 100644 --- a/format/FlightSql.proto +++ b/format/FlightSql.proto @@ -1823,6 +1823,9 @@ message CommandStatementIngest { optional bool temporary = 5; // Perform the ingestion as part of this transaction (if unset, the query is auto-committed). optional bool transaction_id = 6; + + // Future extensions to the parameters of CommandStatementIngest should be added here, at a lower index than the generic 'options' parameter. + // Backend-specific options. map options = 1000; } From 3cdaeacf891ca07f7d6a89623ade13ff5841e7ab Mon Sep 17 00:00:00 2001 From: Joel Lubinitsky Date: Sat, 21 Oct 2023 10:05:23 -0400 Subject: [PATCH 5/5] Fix type of transaction_id --- format/FlightSql.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/format/FlightSql.proto b/format/FlightSql.proto index fd141b7e2fb..56a7d8c5401 100644 --- a/format/FlightSql.proto +++ b/format/FlightSql.proto @@ -1822,7 +1822,7 @@ message CommandStatementIngest { // Use a temporary table. optional bool temporary = 5; // Perform the ingestion as part of this transaction (if unset, the query is auto-committed). - optional bool transaction_id = 6; + optional bytes transaction_id = 6; // Future extensions to the parameters of CommandStatementIngest should be added here, at a lower index than the generic 'options' parameter.