From 3061d9d9d532852d6c6adeddfc0662c64cc6eb35 Mon Sep 17 00:00:00 2001 From: Adam Curtis Date: Tue, 13 Feb 2024 14:17:25 -0500 Subject: [PATCH 01/11] GH-37720: [Format][Docs][FlightSQL] Document FlightSQL stateless prepared statements --- docs/source/format/FlightSql.rst | 3 +++ .../format/FlightSql/CommandPreparedStatementQuery.mmd | 2 ++ .../FlightSql/CommandPreparedStatementQuery.mmd.svg | 2 +- format/FlightSql.proto | 9 +++++++++ 4 files changed, 15 insertions(+), 1 deletion(-) diff --git a/docs/source/format/FlightSql.rst b/docs/source/format/FlightSql.rst index add044c2d362..9013afaa453d 100644 --- a/docs/source/format/FlightSql.rst +++ b/docs/source/format/FlightSql.rst @@ -141,6 +141,9 @@ the ``type`` should be ``ClosePreparedStatement``). Execute a previously created prepared statement and get the results. When used with DoPut: binds parameter values to the prepared statement. + The server may optionally respond with an updated handle. The client + should use this updated handle for all subsequent requests for this + prepared statement. When used with GetFlightInfo: execute the prepared statement. The prepared statement can be reused after fetching results. diff --git a/docs/source/format/FlightSql/CommandPreparedStatementQuery.mmd b/docs/source/format/FlightSql/CommandPreparedStatementQuery.mmd index cb50522eb5a3..cbd1eb6014bc 100644 --- a/docs/source/format/FlightSql/CommandPreparedStatementQuery.mmd +++ b/docs/source/format/FlightSql/CommandPreparedStatementQuery.mmd @@ -28,6 +28,8 @@ Server->>Client: ActionCreatePreparedStatementResult{handle} loop for each invocation of the prepared statement Client->>Server: DoPut(CommandPreparedStatementQuery) Client->>Server: stream of FlightData +Server-->>Client: DoPutPreparedStatementResult{handle} +Note over Client,Server: optional response with updated handle Client->>Server: GetFlightInfo(CommandPreparedStatementQuery) Server->>Client: FlightInfo{endpoints: [FlightEndpoint{…}, …]} loop for each endpoint in FlightInfo.endpoints diff --git a/docs/source/format/FlightSql/CommandPreparedStatementQuery.mmd.svg b/docs/source/format/FlightSql/CommandPreparedStatementQuery.mmd.svg index 96a5bc368829..cbf6a78e9a5c 100644 --- a/docs/source/format/FlightSql/CommandPreparedStatementQuery.mmd.svg +++ b/docs/source/format/FlightSql/CommandPreparedStatementQuery.mmd.svg @@ -1 +1 @@ -ClientServerDoAction(ActionCreatePreparedStatementRequest)1ActionCreatePreparedStatementResult{handle}2DoPut(CommandPreparedStatementQuery)3stream of FlightData4GetFlightInfo(CommandPreparedStatementQuery)5FlightInfo{endpoints: [FlightEndpoint{…}, …]}6DoGet(endpoint.ticket)7stream of FlightData8loop[for each endpoint in FlightInfo.endpoints]loop[for each invocation of the prepared statement]DoAction(ActionClosePreparedStatementRequest)9ActionClosePreparedStatementRequest{}10ClientServer \ No newline at end of file +ServerClientServerClientoptional response with updated handleloop[for each endpoint in FlightInfo.endpoints]loop[for each invocation of the prepared statement]DoAction(ActionCreatePreparedStatementRequest)1ActionCreatePreparedStatementResult{handle}2DoPut(CommandPreparedStatementQuery)3stream of FlightData4DoPutPreparedStatementResult{handle}5GetFlightInfo(CommandPreparedStatementQuery)6FlightInfo{endpoints: [FlightEndpoint{…}, …]}7DoGet(endpoint.ticket)8stream of FlightData9DoAction(ActionClosePreparedStatementRequest)10ActionClosePreparedStatementRequest{}11 \ No newline at end of file diff --git a/format/FlightSql.proto b/format/FlightSql.proto index 581cf1f76d57..406af9d3e0aa 100644 --- a/format/FlightSql.proto +++ b/format/FlightSql.proto @@ -1797,6 +1797,15 @@ message DoPutUpdateResult { int64 record_count = 1; } +// An optional response returned when `DoPut` is called with `CommandPreparedStatementQuery` +message DoPutPreparedStatementResult { + option (experimental) = true; + + // (potentially updated) opaque handle for the prepared statement on the server. + // All subsequent requests for his prepared statement must use this new handle, if specified + bytes prepared_statement_handle = 1; +} + /* * Request message for the "CancelQuery" action. * From cfce010d73e55642d00f7777b6fd606fa2d94c29 Mon Sep 17 00:00:00 2001 From: Adam Curtis Date: Sat, 2 Mar 2024 14:10:28 -0500 Subject: [PATCH 02/11] Fix typo in protobuf docs Co-authored-by: David Li --- format/FlightSql.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/format/FlightSql.proto b/format/FlightSql.proto index 406af9d3e0aa..54f1e8fef6e2 100644 --- a/format/FlightSql.proto +++ b/format/FlightSql.proto @@ -1802,7 +1802,7 @@ message DoPutPreparedStatementResult { option (experimental) = true; // (potentially updated) opaque handle for the prepared statement on the server. - // All subsequent requests for his prepared statement must use this new handle, if specified + // All subsequent requests for this prepared statement must use this new handle, if specified bytes prepared_statement_handle = 1; } From dc96ef7d707d909254fdb0baa6abe4679806fe76 Mon Sep 17 00:00:00 2001 From: Adam Curtis Date: Mon, 4 Mar 2024 09:53:55 -0500 Subject: [PATCH 03/11] Update docs/source/format/FlightSql.rst Co-authored-by: Andrew Lamb --- docs/source/format/FlightSql.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/source/format/FlightSql.rst b/docs/source/format/FlightSql.rst index 9013afaa453d..84bd12014f0b 100644 --- a/docs/source/format/FlightSql.rst +++ b/docs/source/format/FlightSql.rst @@ -143,7 +143,11 @@ the ``type`` should be ``ClosePreparedStatement``). When used with DoPut: binds parameter values to the prepared statement. The server may optionally respond with an updated handle. The client should use this updated handle for all subsequent requests for this - prepared statement. + prepared statement. The updated handle allows implementing query + parameters with stateless services. Note that the server is responsible + for detecting the case where the client does not use the updated handle on + subsequent requests (older clients may ignore this field) and responding + appropriately. When used with GetFlightInfo: execute the prepared statement. The prepared statement can be reused after fetching results. From d4a30c174408f2c93254cc58bc6d314a50bb7e3e Mon Sep 17 00:00:00 2001 From: Adam Curtis Date: Mon, 4 Mar 2024 09:55:40 -0500 Subject: [PATCH 04/11] Update format/FlightSql.proto Co-authored-by: Andrew Lamb --- format/FlightSql.proto | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/format/FlightSql.proto b/format/FlightSql.proto index 54f1e8fef6e2..f41554c40dad 100644 --- a/format/FlightSql.proto +++ b/format/FlightSql.proto @@ -1802,7 +1802,9 @@ message DoPutPreparedStatementResult { option (experimental) = true; // (potentially updated) opaque handle for the prepared statement on the server. - // All subsequent requests for this prepared statement must use this new handle, if specified + // All subsequent requests for this prepared statement must use this new handle, if specified. + // The updated handle allows implementing query parameters with stateless services + // as described in https://github.com/apache/arrow/issues/37720 bytes prepared_statement_handle = 1; } From 0f4d1f446e368b51f5eef0764994f4c35fa2c1f7 Mon Sep 17 00:00:00 2001 From: Adam Curtis Date: Mon, 4 Mar 2024 10:33:38 -0500 Subject: [PATCH 05/11] clarify usage of the updated statement handle --- format/FlightSql.proto | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/format/FlightSql.proto b/format/FlightSql.proto index f41554c40dad..667fc85569c1 100644 --- a/format/FlightSql.proto +++ b/format/FlightSql.proto @@ -1797,15 +1797,24 @@ message DoPutUpdateResult { int64 record_count = 1; } -// An optional response returned when `DoPut` is called with `CommandPreparedStatementQuery` +// An *optional* response returned when `DoPut` is called with `CommandPreparedStatementQuery`. +// +// *Note on legacy behavior*: previous versions of the protocol did not return any result for +// this command, and that behavior should still be supported by clients. See documentation +// of individual fields for more details on expected client behavior in this case. message DoPutPreparedStatementResult { option (experimental) = true; - // (potentially updated) opaque handle for the prepared statement on the server. - // All subsequent requests for this prepared statement must use this new handle, if specified. + // Represents a (potentially updated) opaque handle for the prepared statement on the server. + // Because the handle could potentially be updated, any previous handles for this prepared + // statement should be considered invalid, and all subsequent requests for this prepared + // statement must use this new handle, if specified. // The updated handle allows implementing query parameters with stateless services - // as described in https://github.com/apache/arrow/issues/37720 - bytes prepared_statement_handle = 1; + // as described in https://github.com/apache/arrow/issues/37720. + // + // When an updated handle is not provided by the server, clients should contiue + // using the previous handle provided by `ActionCreatePreparedStatementResonse`. + optional bytes prepared_statement_handle = 1; } /* From bb2d1550afe55b1bab5dbd0b388f5e228fb2b0e5 Mon Sep 17 00:00:00 2001 From: Adam Curtis Date: Mon, 4 Mar 2024 12:39:42 -0500 Subject: [PATCH 06/11] make comment style consistent in flightsql proto --- format/FlightSql.proto | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/format/FlightSql.proto b/format/FlightSql.proto index 667fc85569c1..46c601698bef 100644 --- a/format/FlightSql.proto +++ b/format/FlightSql.proto @@ -1797,11 +1797,12 @@ message DoPutUpdateResult { int64 record_count = 1; } -// An *optional* response returned when `DoPut` is called with `CommandPreparedStatementQuery`. -// -// *Note on legacy behavior*: previous versions of the protocol did not return any result for -// this command, and that behavior should still be supported by clients. See documentation -// of individual fields for more details on expected client behavior in this case. +/* An *optional* response returned when `DoPut` is called with `CommandPreparedStatementQuery`. + * + * *Note on legacy behavior*: previous versions of the protocol did not return any result for + * this command, and that behavior should still be supported by clients. See documentation + * of individual fields for more details on expected client behavior in this case. + */ message DoPutPreparedStatementResult { option (experimental) = true; From 8ea3ca0ef2125f925fbef0d5e6ae735c4645e162 Mon Sep 17 00:00:00 2001 From: Adam Curtis Date: Fri, 15 Mar 2024 20:46:59 -0400 Subject: [PATCH 07/11] Update format/FlightSql.proto Co-authored-by: Andrew Lamb --- format/FlightSql.proto | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/format/FlightSql.proto b/format/FlightSql.proto index 46c601698bef..de2b21fc2427 100644 --- a/format/FlightSql.proto +++ b/format/FlightSql.proto @@ -1810,8 +1810,7 @@ message DoPutPreparedStatementResult { // Because the handle could potentially be updated, any previous handles for this prepared // statement should be considered invalid, and all subsequent requests for this prepared // statement must use this new handle, if specified. - // The updated handle allows implementing query parameters with stateless services - // as described in https://github.com/apache/arrow/issues/37720. + // The updated handle allows implementing query parameters with stateless services. // // When an updated handle is not provided by the server, clients should contiue // using the previous handle provided by `ActionCreatePreparedStatementResonse`. From e0730019f5a7638ee12f0eb6fb10ad4a4a8d8f09 Mon Sep 17 00:00:00 2001 From: Adam Curtis Date: Fri, 15 Mar 2024 20:48:45 -0400 Subject: [PATCH 08/11] Update format/FlightSql.proto Co-authored-by: Andrew Lamb --- format/FlightSql.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/format/FlightSql.proto b/format/FlightSql.proto index de2b21fc2427..3c244cb1e700 100644 --- a/format/FlightSql.proto +++ b/format/FlightSql.proto @@ -1809,7 +1809,7 @@ message DoPutPreparedStatementResult { // Represents a (potentially updated) opaque handle for the prepared statement on the server. // Because the handle could potentially be updated, any previous handles for this prepared // statement should be considered invalid, and all subsequent requests for this prepared - // statement must use this new handle, if specified. + // statement must use this new handle. // The updated handle allows implementing query parameters with stateless services. // // When an updated handle is not provided by the server, clients should contiue From d4dd713f0d98fc433f6c6bfcf40da6f5564d76a4 Mon Sep 17 00:00:00 2001 From: Adam Curtis Date: Fri, 15 Mar 2024 20:50:05 -0400 Subject: [PATCH 09/11] add additional clarifications about usage of stateless parameters --- docs/source/format/FlightSql.rst | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/docs/source/format/FlightSql.rst b/docs/source/format/FlightSql.rst index 84bd12014f0b..d808d25d86eb 100644 --- a/docs/source/format/FlightSql.rst +++ b/docs/source/format/FlightSql.rst @@ -141,13 +141,21 @@ the ``type`` should be ``ClosePreparedStatement``). Execute a previously created prepared statement and get the results. When used with DoPut: binds parameter values to the prepared statement. - The server may optionally respond with an updated handle. The client - should use this updated handle for all subsequent requests for this - prepared statement. The updated handle allows implementing query - parameters with stateless services. Note that the server is responsible - for detecting the case where the client does not use the updated handle on - subsequent requests (older clients may ignore this field) and responding - appropriately. + The server may optionally provide an updated handle in the response. + Updating the handle allows the client to supply all state required to + execute the query in an ActionPreparedStatementExecute message. + For example, stateless servers can encode the bound parameter values into + the new handle, and the client will send that new handle with parameters + back to the server. + + Note that a handle returned from a DoPut call with + CommandPreparedStatementQuery can itself be passed to a subsequent DoPut + call with CommandPreparedStatementQuery to bind a new set of parameters. + The subsequent call itself may return an updated handled which again should + be used for subsequent requests. + + The server is responsible for detecting the case where the client does not + use the updated handle and should return an error. When used with GetFlightInfo: execute the prepared statement. The prepared statement can be reused after fetching results. From 8ddc3c302fabbb95fa78f66c49b8d06dfa5a6bb2 Mon Sep 17 00:00:00 2001 From: Adam Curtis Date: Fri, 15 Mar 2024 21:03:51 -0400 Subject: [PATCH 10/11] clarify legacy behavior for DoPutPreparedResult protobuf message --- format/FlightSql.proto | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/format/FlightSql.proto b/format/FlightSql.proto index 3c244cb1e700..3282ee4f4730 100644 --- a/format/FlightSql.proto +++ b/format/FlightSql.proto @@ -1800,8 +1800,8 @@ message DoPutUpdateResult { /* An *optional* response returned when `DoPut` is called with `CommandPreparedStatementQuery`. * * *Note on legacy behavior*: previous versions of the protocol did not return any result for - * this command, and that behavior should still be supported by clients. See documentation - * of individual fields for more details on expected client behavior in this case. + * this command, and that behavior should still be supported by clients. In that case, the client + * can continue as though the fields in this message were not provided or set to sensible default values. */ message DoPutPreparedStatementResult { option (experimental) = true; From 6cffe2fb3e2e5086f934e5f3df0d1d0737656b37 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Fri, 22 Mar 2024 10:13:41 -0400 Subject: [PATCH 11/11] Update docs/source/format/FlightSql.rst Co-authored-by: Sutou Kouhei --- docs/source/format/FlightSql.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/format/FlightSql.rst b/docs/source/format/FlightSql.rst index dc7f0b9f2d71..5573c0040761 100644 --- a/docs/source/format/FlightSql.rst +++ b/docs/source/format/FlightSql.rst @@ -151,7 +151,7 @@ the ``type`` should be ``ClosePreparedStatement``). Note that a handle returned from a DoPut call with CommandPreparedStatementQuery can itself be passed to a subsequent DoPut call with CommandPreparedStatementQuery to bind a new set of parameters. - The subsequent call itself may return an updated handled which again should + The subsequent call itself may return an updated handle which again should be used for subsequent requests. The server is responsible for detecting the case where the client does not