diff --git a/bittensor/dendrite.py b/bittensor/dendrite.py index 5b72f2f6ae..10fe2a91ab 100644 --- a/bittensor/dendrite.py +++ b/bittensor/dendrite.py @@ -289,6 +289,7 @@ def preprocess_synapse_for_request( Returns: bt.Synapse: The preprocessed synapse. """ + bt.logging.trace("Pre-process synapse for request") # Set the timeout for the synapse synapse.timeout = str(timeout) @@ -331,6 +332,8 @@ def process_server_response(self, server_response, local_synapse: bt.Synapse): Raises: None, but errors in attribute setting are silently ignored. """ + bt.logging.trace("Postprocess server response") + # Check if the server responded with a successful status code if server_response.status_code == 200: # If the response is successful, overwrite local synapse state with diff --git a/bittensor/synapse.py b/bittensor/synapse.py index f5abbe785d..392b24f948 100644 --- a/bittensor/synapse.py +++ b/bittensor/synapse.py @@ -18,7 +18,8 @@ import ast import sys -import pickle +import torch +import json import base64 import typing import hashlib @@ -551,9 +552,17 @@ def to_headers(self) -> dict: headers[f"bt_header_dict_tensor_{field}"] = str(serialized_dict_tensor) elif required and field in required: - serialized_value = pickle.dumps(value) - encoded_value = base64.b64encode(serialized_value).decode("utf-8") - headers[f"bt_header_input_obj_{field}"] = encoded_value + bittensor.logging.trace(f"Serializing {field} with json...") + try: + serialized_value = json.dumps(value) + encoded_value = base64.b64encode(serialized_value.encode()).decode( + "utf-8" + ) + headers[f"bt_header_input_obj_{field}"] = encoded_value + except TypeError as e: + raise ValueError( + f"Error serializing {field} with value {value}. Objects must be json serializable." + ) from e # Adding the size of the headers and the total size to the headers headers["header_size"] = str(sys.getsizeof(headers)) @@ -647,15 +656,21 @@ def parse_headers_to_inputs(cls, headers: dict) -> dict: continue # Handle 'input_obj' headers elif "bt_header_input_obj" in key: + bittensor.logging.trace(f"Deserializing {key} with json...") try: new_key = key.split("bt_header_input_obj_")[1] # Skip if the key already exists in the dictionary if new_key in inputs_dict: continue # Decode and load the serialized object - inputs_dict[new_key] = pickle.loads( - base64.b64decode(value.encode("utf-8")) + inputs_dict[new_key] = json.loads( + base64.b64decode(value.encode()).decode("utf-8") ) + except json.JSONDecodeError as e: + bittensor.logging.error( + f"Error while json decoding 'input_obj' header {key}: {e}" + ) + continue except Exception as e: bittensor.logging.error( f"Error while parsing 'input_obj' header {key}: {e}" diff --git a/requirements/prod.txt b/requirements/prod.txt index 47970a6b35..82f28437af 100644 --- a/requirements/prod.txt +++ b/requirements/prod.txt @@ -11,7 +11,7 @@ munch==2.5.0 netaddr numpy msgpack -msgpack_numpy +git+https://github.com/opentensor/msgpack-numpy.git#egg=msgpack-numpy nest_asyncio pycryptodome>=3.18.0,<4.0.0 pyyaml diff --git a/scripts/check_compatibility.sh b/scripts/check_compatibility.sh index 3a6cf47e4c..5f48f4cbb0 100755 --- a/scripts/check_compatibility.sh +++ b/scripts/check_compatibility.sh @@ -17,6 +17,11 @@ check_compatibility() { all_supported=0 while read -r requirement; do + # Skip lines starting with git+ + if [[ "$requirement" == git+* ]]; then + continue + fi + package_name=$(echo "$requirement" | awk -F'[!=<>]' '{print $1}' | awk -F'[' '{print $1}') # Strip off brackets echo -n "Checking $package_name... " diff --git a/setup.py b/setup.py index e8354d8d80..654acee9d4 100644 --- a/setup.py +++ b/setup.py @@ -26,22 +26,46 @@ import subprocess +class SubmoduleSyncError(Exception): + pass + + def sync_and_update_submodules(): try: print("Synchronizing and updating submodules...") - subprocess.check_call(['git', 'submodule', 'sync']) - subprocess.check_call(['git', 'submodule', 'update', '--init']) + subprocess.check_call(["git", "submodule", "sync"]) + subprocess.check_call(["git", "submodule", "update", "--init"]) except subprocess.CalledProcessError: - print("Error synchronizing or updating submodules. Please ensure you have git installed and are in the root directory of the repository.") - raise + print( + "Error synchronizing or updating submodules. Please ensure you have git installed and are in the root directory of the repository." + ) + raise SubmoduleSyncError( + "An error occurred while synchronizing or updating submodules." + ) + + +try: + sync_and_update_submodules() +except SubmoduleSyncError as e: + print(f"Submodule synchronization error: {e}") -sync_and_update_submodules() def read_requirements(path): + requirements = [] + git_requirements = [] + with pathlib.Path(path).open() as requirements_txt: - return [ - str(requirement) for requirement in parse_requirements(requirements_txt) - ] + for line in requirements_txt: + if line.startswith("git+"): + git_requirements.append(line.strip()) + else: + requirements.append(line.strip()) + + # Install git dependencies + for git_req in git_requirements: + subprocess.check_call(["pip", "install", git_req]) + + return requirements requirements = read_requirements("requirements/prod.txt") diff --git a/tests/unit_tests/test_synapse.py b/tests/unit_tests/test_synapse.py index a44c0d92c6..68f2d9541b 100644 --- a/tests/unit_tests/test_synapse.py +++ b/tests/unit_tests/test_synapse.py @@ -14,8 +14,8 @@ # THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER # DEALINGS IN THE SOFTWARE. +import json import torch -import pickle import base64 import typing import pytest @@ -31,9 +31,9 @@ class Test(bittensor.Synapse): headers = { "bt_header_axon_nonce": "111", "bt_header_dendrite_ip": "12.1.1.2", - "bt_header_input_obj_key1": base64.b64encode(pickle.dumps([1, 2, 3, 4])).decode( - "utf-8" - ), + "bt_header_input_obj_key1": base64.b64encode( + json.dumps([1, 2, 3, 4]).encode("utf-8") + ).decode("utf-8"), "bt_header_tensor_key2": "[3]-torch.float32", "timeout": "12", "name": "Test", @@ -66,9 +66,9 @@ class Test(bittensor.Synapse): headers = { "bt_header_axon_nonce": "111", "bt_header_dendrite_ip": "12.1.1.2", - "bt_header_input_obj_key1": base64.b64encode(pickle.dumps([1, 2, 3, 4])).decode( - "utf-8" - ), + "bt_header_input_obj_key1": base64.b64encode( + json.dumps([1, 2, 3, 4]).encode("utf-8") + ).decode("utf-8"), "bt_header_tensor_key2": "[3]-torch.float32", "timeout": "12", "name": "Test",