diff --git a/README.md b/README.md index bf28b71..2712dcf 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # e6data Python Connector -![version](https://img.shields.io/badge/version-1.0.8-blue.svg) +![version](https://img.shields.io/badge/version-1.0.9-blue.svg) ## Introduction @@ -68,6 +68,13 @@ limit = 500 records = cursor.fetchmany(limit) ``` +To fetch all the records in buffer to reduce memory consumption: +```python +records_iterator = cursor.fetchall_buffer() # Returns generator +for item in records_iterator: + print(item) +``` + To get the execution plan after query execution: ```python import json diff --git a/e6xdb/e6x.py b/e6xdb/e6x.py index ffcf971..ed21e6c 100644 --- a/e6xdb/e6x.py +++ b/e6xdb/e6x.py @@ -425,6 +425,15 @@ def _fetch_all(self): self._data = None return rows + def fetchall_buffer(self, query_id=None): + if query_id: + self._query_id = query_id + while True: + rows = self.fetch_batch() + if not rows: + return + yield rows + def fetch_batch(self): # _logger.debug("fetching next batch from e6data") client = self.connection.client @@ -439,11 +448,15 @@ def fetch_batch(self): # one batch retrieves the predefined set of rows return read_rows_from_batch(self._query_columns_description, dis) - def fetchall(self): + def fetchall(self, query_id=None): + if query_id: + self._query_id = query_id return self._fetch_all() - def fetchmany(self, size=None): + def fetchmany(self, size=None, query_id=None): # _logger.info("fetching all from overriden method") + if query_id: + self._query_id = query_id if size is None: size = self.arraysize if self._data is None: diff --git a/setup.py b/setup.py index 2cd8caf..079afdb 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ envstring = lambda var: os.environ.get(var) or "" -VERSION = [1, 0, 8] +VERSION = [1, 0, 9] def get_long_desc():