From aaab83c5a570b989e05a3af6542ad5c992e31a33 Mon Sep 17 00:00:00 2001 From: Edward Park Date: Mon, 26 Feb 2024 19:40:16 -0500 Subject: [PATCH 1/5] Add S3_ENDPOINT to customize S3 endpoint URL --- CHANGELOG.md | 15 ++++++++++----- nucleus/utils.py | 6 +++++- pyproject.toml | 2 +- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d4228591..b127f64e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,16 @@ All notable changes to the [Nucleus Python Client](https://github.com/scaleapi/n The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.17.2] - 2024-02-26 + +### Added +- Added the environment variable `S3_ENDPOINT` to accomodate for nonstandard S3 Endpoint URLs when asking for presigned URLs + +## [0.17.1] - 2024-02-22 + +### Added +- Added the environment variable `NUCLEUS_SKIP_SSL_VERIFY` to skip SSL verification on requests + ## [0.17.0](https://github.com/scaleapi/nucleus-python-client/releases/tag/v0.17.0) - 2024-02-06 ### Added @@ -14,11 +24,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixes - Fix test `test_models.test_remove_invalid_tag_from_model` -## [0.17.1] - 2024-02-22 - -### Added -- Environment variable `NUCLEUS_SKIP_SSL_VERIFY` to skip SSL verification on requests - ## [0.16.18](https://github.com/scaleapi/nucleus-python-client/releases/tag/v0.16.18) - 2024-02-06 ### Added diff --git a/nucleus/utils.py b/nucleus/utils.py index 4a7f06a3..9d03e7c7 100644 --- a/nucleus/utils.py +++ b/nucleus/utils.py @@ -3,6 +3,7 @@ import io import json import os +import urllib.request import uuid from collections import defaultdict from typing import IO, TYPE_CHECKING, Dict, List, Sequence, Tuple, Type, Union @@ -381,9 +382,12 @@ def serialize_and_write_to_presigned_url( ): """This helper function can be used to serialize a list of API objects to NDJSON.""" request_id = uuid.uuid4().hex + route = f"dataset/{dataset_id}/signedUrl/{request_id}" + if (os.environ.get("S3_ENDPOINT") is not None): + route += "?s3Endpoint=" + urllib.request.pathname2url(os.environ.get("S3_ENDPOINT")) response = client.make_request( payload={}, - route=f"dataset/{dataset_id}/signedUrl/{request_id}", + route=route, requests_command=requests.get, ) diff --git a/pyproject.toml b/pyproject.toml index d4a2bf78..ce7d6797 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ ignore = ["E501", "E741", "E731", "F401"] # Easy ignore for getting it running [tool.poetry] name = "scale-nucleus" -version = "0.17.1" +version = "0.17.2" description = "The official Python client library for Nucleus, the Data Platform for AI" license = "MIT" authors = ["Scale AI Nucleus Team "] From b14d9ddc5e1be77dff6a5b0ba6fa58ea49a56092 Mon Sep 17 00:00:00 2001 From: Edward Park Date: Mon, 26 Feb 2024 19:53:44 -0500 Subject: [PATCH 2/5] formatting --- nucleus/utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/nucleus/utils.py b/nucleus/utils.py index 9d03e7c7..0d15d81e 100644 --- a/nucleus/utils.py +++ b/nucleus/utils.py @@ -383,8 +383,10 @@ def serialize_and_write_to_presigned_url( """This helper function can be used to serialize a list of API objects to NDJSON.""" request_id = uuid.uuid4().hex route = f"dataset/{dataset_id}/signedUrl/{request_id}" - if (os.environ.get("S3_ENDPOINT") is not None): - route += "?s3Endpoint=" + urllib.request.pathname2url(os.environ.get("S3_ENDPOINT")) + if os.environ.get("S3_ENDPOINT") is not None: + route += "?s3Endpoint=" + urllib.request.pathname2url( + os.environ.get("S3_ENDPOINT") + ) response = client.make_request( payload={}, route=route, From bad22dec77caffa4d8e5d2f4646a73a426ab0f6b Mon Sep 17 00:00:00 2001 From: Edward Park Date: Mon, 26 Feb 2024 20:47:06 -0500 Subject: [PATCH 3/5] type checking --- nucleus/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nucleus/utils.py b/nucleus/utils.py index 0d15d81e..b59d0fdf 100644 --- a/nucleus/utils.py +++ b/nucleus/utils.py @@ -385,7 +385,7 @@ def serialize_and_write_to_presigned_url( route = f"dataset/{dataset_id}/signedUrl/{request_id}" if os.environ.get("S3_ENDPOINT") is not None: route += "?s3Endpoint=" + urllib.request.pathname2url( - os.environ.get("S3_ENDPOINT") + os.environ["S3_ENDPOINT"] ) response = client.make_request( payload={}, From 22d54d7dfa7576d34650da5aec5b1cb70eb4bc55 Mon Sep 17 00:00:00 2001 From: Sam Videlock Date: Thu, 29 Feb 2024 10:12:38 -0500 Subject: [PATCH 4/5] bumping to 0.17.3 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index ce7d6797..66ab6406 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ ignore = ["E501", "E741", "E731", "F401"] # Easy ignore for getting it running [tool.poetry] name = "scale-nucleus" -version = "0.17.2" +version = "0.17.3" description = "The official Python client library for Nucleus, the Data Platform for AI" license = "MIT" authors = ["Scale AI Nucleus Team "] From efa08c750954bd15956a82dd0ce2a39b63656c2b Mon Sep 17 00:00:00 2001 From: Sam Videlock Date: Thu, 29 Feb 2024 10:48:58 -0500 Subject: [PATCH 5/5] updading date on change log --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e39c23e9..06df7d40 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to the [Nucleus Python Client](https://github.com/scaleapi/n The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [0.17.3] - 2024-02-26 +## [0.17.3] - 2024-02-29 ### Added - Added the environment variable `S3_ENDPOINT` to accomodate for nonstandard S3 Endpoint URLs when asking for presigned URLs