From 0239f97325966e2b8eee20f222574fd88bd78b97 Mon Sep 17 00:00:00 2001 From: D059251 Date: Wed, 30 Sep 2020 16:49:17 +0200 Subject: [PATCH 1/7] fix(batch): encode batch req body with utf-8 --- .gitignore | 1 + odata/batchcontext.py | 2 +- odata/connection.py | 1 - 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 1eb011e..b99c066 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ dist build/ doc/_build/ venv/ +.env \ No newline at end of file diff --git a/odata/batchcontext.py b/odata/batchcontext.py index 44c84ed..dac455e 100644 --- a/odata/batchcontext.py +++ b/odata/batchcontext.py @@ -118,7 +118,7 @@ def _get_payload(self): parts_str.append(pl) parts_str.append('--%s--' % self.boundary) - return '\n'.join(parts_str) + return '\n'.join(parts_str).replace('\n', '\r\n').encode('utf-8') def query(self, entitycls): diff --git a/odata/connection.py b/odata/connection.py index 5571680..1160f5b 100644 --- a/odata/connection.py +++ b/odata/connection.py @@ -157,7 +157,6 @@ def execute_post_raw(self, url, headers, data: str, params=None): self.log.info(u'POST {0}'.format(url)) self.log.info(u'Payload: {0}'.format(data)) - data = data.replace('\n', '\r\n') response = self._do_post(url, data=data, headers=headers, params=params) self._handle_odata_error(response) response_ct = response.headers.get('content-type', '') From 1b007dd537d9876ee25a62cbce17961b870e8a7a Mon Sep 17 00:00:00 2001 From: D059251 Date: Wed, 7 Oct 2020 09:32:28 +0200 Subject: [PATCH 2/7] feat: encode urls in batch payload --- odata/changeset.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/odata/changeset.py b/odata/changeset.py index dd3636d..861d109 100644 --- a/odata/changeset.py +++ b/odata/changeset.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- +import urllib import json import enum from uuid import uuid4 as uuid @@ -40,7 +41,8 @@ def get_payload(self): parts.append('%s: %s' % (key, value)) parts.append('') - parts.append('%s %s HTTP/1.1' % (self.method, self.url)) + url_encoded = urllib.parse.quote(self.url) + parts.append('%s %s HTTP/1.1' % (self.method, url_encoded)) parts.append('Host: %s' % socket.gethostname()) parts.append('Content-Type: application/json;type=entry') parts.append('') From 36701c390ef9ea58fa4968f207408a1d8f6d7581 Mon Sep 17 00:00:00 2001 From: D059251 Date: Thu, 8 Oct 2020 13:32:42 +0200 Subject: [PATCH 3/7] feat: basic support for calling action/function in batch request --- odata/batchcontext.py | 30 +++++++++++++++-- odata/changeset.py | 75 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 102 insertions(+), 3 deletions(-) diff --git a/odata/batchcontext.py b/odata/batchcontext.py index dac455e..799078d 100644 --- a/odata/batchcontext.py +++ b/odata/batchcontext.py @@ -3,8 +3,9 @@ from odata.query import Query from odata.context import Context from odata.exceptions import ODataError +from odata.action import Action, Function from uuid import uuid4 as uuid -from odata.changeset import ChangeSet, Change, ChangeAction +from odata.changeset import ChangeSet, Change, ChangeAction, ActionChange, FunctionChange class BatchContext(Context): def __init__(self, service, session=None, auth=None): @@ -67,6 +68,7 @@ def _apply_response_to_entities(self, response, content_id_to_entity_map): m = content_id_to_entity_map entities = [] response_map = [] + processed_content_ids = [] for entity, content_id in m: saved_data = {} error_msg = None @@ -77,6 +79,7 @@ def _apply_response_to_entities(self, response, content_id_to_entity_map): error_code = 500 error_msg = 'Server sent no error message. There might be errors in previous operations of the same batch.' else: + processed_content_ids.append(content_id) resp_for_entity = resp_for_entity[0] if resp_for_entity['status'] < 200 or resp_for_entity['status'] >= 300: @@ -104,6 +107,18 @@ def _apply_response_to_entities(self, response, content_id_to_entity_map): entities.append(entity) + for res in [x for x in response['responses'] if x['id'] not in processed_content_ids]: + if res['status'] < 200 or res['status'] >= 300: + error_code = res['status'] + error_msg = "HTTP %s for content_id '%s' with error %s" % ( + res['status'], + res['id'], + res.get('body', {}).get('error', {}).get('message', 'Server sent no error message') + ) + response_map.append((None, error_code, error_msg)) + else: + response_map.append((None, res['status'], None)) + return { 'entities': entities, 'response_map': response_map, @@ -128,8 +143,17 @@ def query(self, entitycls): # q = Query(entitycls, connection=self.connection) # return q - def call(self, action_or_function, **parameters): - raise NotImplementedError('calling an action/function in a batch operation is not implemented') + def call(self, action_or_function, callback=None, **parameters): + if self._changeset is None: + raise Exception('Call open_changeset before doing data modification requests') + if isinstance(action_or_function, Action): + change = ActionChange(action_or_function, **parameters) + self._changeset.add_change(change, callback=callback) + return + elif isinstance(action_or_function, Function): + change = FunctionChange(action_or_function, **parameters) + self._changeset.add_change(change, callback=callback) + return def call_with_query(self, action_or_function, query, **parameters): raise NotImplementedError('calling an action/function with query in a batch operation is not implemented') diff --git a/odata/changeset.py b/odata/changeset.py index 861d109..fb9e56c 100644 --- a/odata/changeset.py +++ b/odata/changeset.py @@ -50,6 +50,81 @@ def get_payload(self): return '\n'.join(parts) +class ActionChange(): + def __init__(self, action, **kwargs): + self.content_id = None + self.base_headers = { + 'Content-Type': 'application/http', + 'Content-Transfer-Encoding': 'binary', + } + self.action = action + self.kwargs = kwargs + + def get_content_id(self): + return self.content_id + + def set_content_id(self, content_id: str): + self.content_id = content_id + return self + + def get_payload(self): + headers = self.base_headers.copy() + headers.update({ + 'Content-ID': self.content_id, + }) + parts = [] + + for key, value in headers.items(): + parts.append('%s: %s' % (key, value)) + parts.append('') + + url = '/' + self.action.name + + parts.append('POST %s HTTP/1.1' % (url)) + parts.append('Host: %s' % socket.gethostname()) + parts.append('Content-Type: application/json;type=entry') + parts.append('') + parts.append(json.dumps(self.kwargs, indent=2, ensure_ascii=False)) + + return '\n'.join(parts) + +class FunctionChange(): + def __init__(self, function): + self.content_id = None + self.base_headers = { + 'Content-Type': 'application/http', + 'Content-Transfer-Encoding': 'binary', + } + self.function = function + + def get_content_id(self): + return self.content_id + + def set_content_id(self, content_id: str): + self.content_id = content_id + return self + + def get_payload(self): + headers = self.base_headers.copy() + headers.update({ + 'Content-ID': self.content_id, + }) + parts = [] + + for key, value in headers.items(): + parts.append('%s: %s' % (key, value)) + parts.append('') + + url = self.function.__odata_service__.url + if not url.endswith('/'): + url += '/' + url += self.function.name + + parts.append('GET %s HTTP/1.1' % (url)) + parts.append('Host: %s' % socket.gethostname()) + parts.append('') + + return '\n'.join(parts) class ChangeSet: def __init__(self): From 8481df077829cb94a5126ce18ee62fd261f9dd50 Mon Sep 17 00:00:00 2001 From: Lukas Riegel Date: Tue, 20 Oct 2020 11:56:14 +0200 Subject: [PATCH 4/7] fix: changeset identifier should not be encoded --- odata/changeset.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/odata/changeset.py b/odata/changeset.py index 35568af..53e298b 100644 --- a/odata/changeset.py +++ b/odata/changeset.py @@ -42,6 +42,10 @@ def get_payload(self): parts.append('') url_encoded = urllib.parse.quote(self.url) + if url_encoded.startswith("%24changeset_"): + url_encoded = url_encoded.replace("%24changeset_", "$changeset_") + + parts.append('%s %s HTTP/1.1' % (self.method, url_encoded)) parts.append('Host: %s' % socket.gethostname()) parts.append('Content-Type: application/json;type=entry;charset=utf-8') From 06d9dc05b97a6f412290f3dc6ab92bdb0fb24161 Mon Sep 17 00:00:00 2001 From: D059251 Date: Tue, 27 Oct 2020 12:05:30 +0100 Subject: [PATCH 5/7] feat: omit null values for FKeys for 1:n deep inserts --- odata/batchcontext.py | 42 +++++++++++++++++++++++++++++------------- odata/connection.py | 6 +++--- 2 files changed, 32 insertions(+), 16 deletions(-) diff --git a/odata/batchcontext.py b/odata/batchcontext.py index d992933..a6143d3 100644 --- a/odata/batchcontext.py +++ b/odata/batchcontext.py @@ -220,24 +220,40 @@ def _insert_new(self, entity, parent_resource=None): else: es_p = parent_resource.__odata__ entity_type = entity.__odata_schema__['type'] - parent_entity_type = parent_resource.__odata_schema__['type'] parent_nav_prop = [x for x in es_p.navigation_properties if x[1].navigated_property_type == entity_type][0][1] - content_id = [x for x in self._content_id_to_entity_map if x[0] is parent_resource][0][1] # use $/navProperty as url url = '$%s/%s' % (content_id, parent_nav_prop.name) - - # via the url we tell odata that we want to create a sub-entity (e.g. Author = parent and Book = sub). - # In case the book has a reference to author (e.g. author_ID) we need to remove it as it has no value and - # defaults to a "null"-value if not set. However, we just dont want to send any value (not even null) for this field - nav_prop = [x for x in es.navigation_properties if x[1].navigated_property_type == parent_entity_type] - if nav_prop and len(nav_prop) > 0: - fk = nav_prop[0][1].foreign_key - if fk is not None and fk in insert_data: - # remove if it exists in the dict - insert_data.pop(fk) - + + # For deep inserts we must not send (even not with "null" as value) the reference keys. E.g. + # Book -> Page -> Text + # When we create a "Text" we must not set the field Page_ID (or however the foreign key is named) in the + # request as the server will automatically fill it in by the magic of deep inserts. + def filter_insert_data(data, entity_schema, parent=None): + parent_type = None if parent is None else parent.__odata_schema__['type'] + + for nav_prop in entity_schema.navigation_properties: + fk = nav_prop[1].foreign_key + fk_exists = fk is not None and fk in data + nested_field = nav_prop[0] + nested_field_set = nested_field is not None and nested_field in data + links_to_parent = False if parent_resource is None else nav_prop[1].navigated_property_type == parent_type + + if fk_exists: + if nested_field_set: + data.pop(fk) + elif links_to_parent: + data.pop(fk) + elif nested_field_set: + nav_prop_cls = nav_prop[1].entitycls() + # recursion for deep inserts via nav properties with a 1:n/1:1 relationship. + filter_insert_data(data[nested_field], nav_prop_cls.__odata__, parent=entity_schema.entity) + + return data + filter_insert_data(insert_data, es, parent=parent_resource) + + if url is None: msg = 'Cannot insert Entity that does not belong to EntitySet: {0}'.format(entity) raise ODataError(msg) diff --git a/odata/connection.py b/odata/connection.py index 84d75d5..f6327a4 100644 --- a/odata/connection.py +++ b/odata/connection.py @@ -142,7 +142,7 @@ def execute_post(self, url, data, params=None): data = json.dumps(data) self.log.info(u'POST {0}'.format(url)) - self.log.info(u'Payload: {0}'.format(data)) + self.log.debug(u'Payload: {0}'.format(data)) response = self._do_post(url, data=data, headers=headers, params=params) self._handle_odata_error(response) @@ -157,7 +157,7 @@ def execute_post_raw(self, url, headers, data: str, params=None): headers = {**ODataConnection.base_headers, **headers} self.log.info(u'POST {0}'.format(url)) - self.log.info(u'Payload: {0}'.format(data)) + self.log.debug(u'Payload: {0}'.format(data)) response = self._do_post(url, data=data, headers=headers, params=params) self._handle_odata_error(response) response_ct = response.headers.get('content-type', '') @@ -176,7 +176,7 @@ def execute_patch(self, url, data): data = json.dumps(data) self.log.info(u'PATCH {0}'.format(url)) - self.log.info(u'Payload: {0}'.format(data)) + self.log.debug(u'Payload: {0}'.format(data)) response = self._do_patch(url, data=data, headers=headers) self._handle_odata_error(response) From ba99dff00110f8edc4b2fe0fa6b3a9f9095fcf9a Mon Sep 17 00:00:00 2001 From: Lukas Riegel Date: Wed, 28 Oct 2020 15:30:13 +0100 Subject: [PATCH 6/7] feat: basi delete functionality --- odata/batchcontext.py | 43 +++++++++++++++++++++++++------------------ 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/odata/batchcontext.py b/odata/batchcontext.py index a6143d3..184eda0 100644 --- a/odata/batchcontext.py +++ b/odata/batchcontext.py @@ -92,15 +92,14 @@ def _apply_response_to_entities(self, response, content_id_to_entity_map): ) if error_msg is None: - saved_data = resp_for_entity['body'] - for k in list(saved_data.keys()): - # remove odata annotations in the response - if k.startswith('@odata.'): - saved_data.pop(k) - - entity.__odata__.reset() # reset dirty flags etc - entity.__odata__.update(saved_data) - + saved_data = resp_for_entity['body'] if 'body' in resp_for_entity else {} + if saved_data: + for k in list(saved_data.keys()): + # remove odata annotations in the response + if k.startswith('@odata.'): + saved_data.pop(k) + entity.__odata__.reset() # reset dirty flags etc + entity.__odata__.update(saved_data) response_map.append((entity, resp_for_entity['status'], None)) else: response_map.append((entity, error_code, error_msg)) @@ -190,16 +189,24 @@ def delete(self, entity): """ if self._changeset is None: raise Exception('Call open_changeset before doing data modification requests') + es = entity.__odata__ + url = entity.__odata__.instance_url[len(self._service.url) - 1:] + + if url is None: + msg = 'Cannot delete Entity that does not belong to EntitySet: {0}'.format(entity) + raise ODataError(msg) - raise Exception('Delete still needs to be implemented') - # TODO: - - # self.log.info(u'Deleting entity: {0}'.format(entity)) - # # url = entity.__odata__.instance_url - # url = entity.__odata__.instance_url[len(self._service.url) - 1:] - # self.connection.execute_delete(url) - # entity.__odata__.persisted = False - # self.log.info(u'Success') + def cb(self, saved_data): + es.reset() + self.log.info(u'Success') + + content_id = self._changeset.add_change(Change( + url, + None, + ChangeAction.DELETE, + ), callback=cb) + self._content_id_to_entity_map.append((entity, content_id)) + return content_id def _insert_new(self, entity, parent_resource=None): """ From 0eecf68c8f357db8a5fa567d29470552fb2877db Mon Sep 17 00:00:00 2001 From: Lukas Riegel Date: Thu, 29 Oct 2020 14:07:14 +0100 Subject: [PATCH 7/7] feat: integer, decimal as strings according to IEEE754Compatible --- odata/changeset.py | 2 +- odata/connection.py | 4 ++-- odata/property.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/odata/changeset.py b/odata/changeset.py index 53e298b..3579b9c 100644 --- a/odata/changeset.py +++ b/odata/changeset.py @@ -48,7 +48,7 @@ def get_payload(self): parts.append('%s %s HTTP/1.1' % (self.method, url_encoded)) parts.append('Host: %s' % socket.gethostname()) - parts.append('Content-Type: application/json;type=entry;charset=utf-8') + parts.append('Content-Type: application/json;type=entry;charset=utf-8;IEEE754Compatible=true') parts.append('') parts.append(json.dumps(self.data, indent=2, ensure_ascii=False)) diff --git a/odata/connection.py b/odata/connection.py index f6327a4..63b99d5 100644 --- a/odata/connection.py +++ b/odata/connection.py @@ -135,7 +135,7 @@ def execute_get(self, url, params=None): def execute_post(self, url, data, params=None): headers = { - 'Content-Type': 'application/json;charset=utf-8', + 'Content-Type': 'application/json;charset=utf-8;IEEE754Compatible=true', } headers.update(self.base_headers) @@ -169,7 +169,7 @@ def execute_post_raw(self, url, headers, data: str, params=None): def execute_patch(self, url, data): headers = { - 'Content-Type': 'application/json;charset=utf-8', + 'Content-Type': 'application/json;charset=utf-8;IEEE754Compatible=true', } headers.update(self.base_headers) diff --git a/odata/property.py b/odata/property.py index 35814ed..6f3fa87 100644 --- a/odata/property.py +++ b/odata/property.py @@ -243,7 +243,7 @@ class FloatProperty(PropertyBase): Property that stores a float value """ def serialize(self, value): - return value + return str(value) def deserialize(self, value): return value @@ -261,7 +261,7 @@ def escape_value(self, value): def serialize(self, value): if value is not None: - return float(value) + return str(float(value)) def deserialize(self, value): if value is not None: