From d751cbea775e870c3e2cdcdc7e72e580db3b1e5d Mon Sep 17 00:00:00 2001 From: Vishal Anand <101251245+vishale6x@users.noreply.github.com> Date: Tue, 6 Jun 2023 10:44:52 +0530 Subject: [PATCH 1/4] fetchall buffer added. --- README.md | 7 +++++++ e6xdb/e6x.py | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/README.md b/README.md index bf28b71..6fe17e3 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,13 @@ limit = 500 records = cursor.fetchmany(limit) ``` +To fetch all the records in buffer to help in memory consumption: +```python +records_iterator = cursor.fetchall_buffer() # Returns generator +for item in records_iterator: + print(item) +``` + To get the execution plan after query execution: ```python import json diff --git a/e6xdb/e6x.py b/e6xdb/e6x.py index ffcf971..d565cec 100644 --- a/e6xdb/e6x.py +++ b/e6xdb/e6x.py @@ -425,6 +425,13 @@ def _fetch_all(self): self._data = None return rows + def fetchall_buffer(self): + while True: + rows = self.fetch_batch() + if not rows: + return + yield rows + def fetch_batch(self): # _logger.debug("fetching next batch from e6data") client = self.connection.client From d7cca6d6392de879691508032195bd9081c1474e Mon Sep 17 00:00:00 2001 From: Vishal Anand <101251245+vishale6x@users.noreply.github.com> Date: Tue, 6 Jun 2023 15:26:52 +0530 Subject: [PATCH 2/4] fetchall buffer added. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6fe17e3..8b0f782 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,7 @@ limit = 500 records = cursor.fetchmany(limit) ``` -To fetch all the records in buffer to help in memory consumption: +To fetch all the records in buffer to reduce memory consumption: ```python records_iterator = cursor.fetchall_buffer() # Returns generator for item in records_iterator: From fa048117d8175a5cae6ad1b824e47fa57350c586 Mon Sep 17 00:00:00 2001 From: Vishal Anand <101251245+vishale6x@users.noreply.github.com> Date: Tue, 6 Jun 2023 18:13:45 +0530 Subject: [PATCH 3/4] fetchall buffer added. --- e6xdb/e6x.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/e6xdb/e6x.py b/e6xdb/e6x.py index d565cec..ed21e6c 100644 --- a/e6xdb/e6x.py +++ b/e6xdb/e6x.py @@ -425,7 +425,9 @@ def _fetch_all(self): self._data = None return rows - def fetchall_buffer(self): + def fetchall_buffer(self, query_id=None): + if query_id: + self._query_id = query_id while True: rows = self.fetch_batch() if not rows: @@ -446,11 +448,15 @@ def fetch_batch(self): # one batch retrieves the predefined set of rows return read_rows_from_batch(self._query_columns_description, dis) - def fetchall(self): + def fetchall(self, query_id=None): + if query_id: + self._query_id = query_id return self._fetch_all() - def fetchmany(self, size=None): + def fetchmany(self, size=None, query_id=None): # _logger.info("fetching all from overriden method") + if query_id: + self._query_id = query_id if size is None: size = self.arraysize if self._data is None: From 2080c9abba68a018bdbdbde1d815af244c764a85 Mon Sep 17 00:00:00 2001 From: Vishal Anand <101251245+vishale6x@users.noreply.github.com> Date: Wed, 7 Jun 2023 11:50:25 +0530 Subject: [PATCH 4/4] fetchall buffer added. --- README.md | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8b0f782..2712dcf 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # e6data Python Connector -![version](https://img.shields.io/badge/version-1.0.8-blue.svg) +![version](https://img.shields.io/badge/version-1.0.9-blue.svg) ## Introduction diff --git a/setup.py b/setup.py index 2cd8caf..079afdb 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ envstring = lambda var: os.environ.get(var) or "" -VERSION = [1, 0, 8] +VERSION = [1, 0, 9] def get_long_desc():