From 2e0d16f74f4b3786ee33e858c5d2aef5d3fac1ea Mon Sep 17 00:00:00 2001 From: Martin Strobel Date: Tue, 14 May 2019 09:49:09 -0700 Subject: [PATCH 1/6] Adding a class to prevent entry into a code-block until a file lock can be taken. This replicates logic in the .NET exensions repository found here: https://github.com/AzureAD/microsoft-authentication-extensions-for-dotnet/blob/113ecc725891395dd7212aac82d0681f9a0d60ce/src/Shared/CrossPlatLock.cs It will allow .NET processes and Python processes to not step on eachother's toes. It does add a dependency on two new libraries: psutil and portalocker --- msal_extensions/__init__.py | 2 ++ msal_extensions/_cache_lock.py | 37 ++++++++++++++++++++++++++++++++++ setup.py | 4 ++++ tests/lock_acquire.py | 28 +++++++++++++++++++++++++ tests/test_crossplatlock.py | 18 +++++++++++++++++ 5 files changed, 89 insertions(+) create mode 100644 msal_extensions/_cache_lock.py create mode 100644 tests/lock_acquire.py create mode 100644 tests/test_crossplatlock.py diff --git a/msal_extensions/__init__.py b/msal_extensions/__init__.py index f102a9c..01da028 100644 --- a/msal_extensions/__init__.py +++ b/msal_extensions/__init__.py @@ -1 +1,3 @@ __version__ = "0.0.1" + +from ._cache_lock import CrossPlatLock diff --git a/msal_extensions/_cache_lock.py b/msal_extensions/_cache_lock.py new file mode 100644 index 0000000..ab85a87 --- /dev/null +++ b/msal_extensions/_cache_lock.py @@ -0,0 +1,37 @@ +import os +import errno +import datetime +import psutil +import portalocker + + +class CrossPlatLock(object): + TIMEOUT = datetime.timedelta(minutes=1) + RETRY_WAIT = datetime.timedelta(milliseconds=100) + RETRY_COUNT = int(TIMEOUT.total_seconds() / RETRY_WAIT.total_seconds()) + + def __init__(self, lockfile_path): + self._lockpath = lockfile_path + + def __enter__(self): + pid = os.getpid() + proc = psutil.Process(pid) + lock_dir = os.path.dirname(self._lockpath) + if not os.path.exists(lock_dir): + os.makedirs(lock_dir) + + self._fh = open(self._lockpath, 'wb+', buffering=0) + portalocker.lock(self._fh, portalocker.LOCK_EX) + self._fh.write('{} {}'.format(pid, proc.name()).encode('utf-8')) + + def __exit__(self, *args): + self._fh.close() + try: + # Attempt to delete the lockfile. In either of the failure cases enumerated below, it is likely that + # another process has raced this one and ended up clearing or locking the file for itself. + os.remove(self._lockpath) + except PermissionError: + pass + except OSError as ex: + if ex.errno != errno.ENOENT: + raise diff --git a/setup.py b/setup.py index 17d9fca..3542ae1 100644 --- a/setup.py +++ b/setup.py @@ -15,5 +15,9 @@ classifiers=[ 'Development Status :: 2 - Pre-Alpha', ], + install_requires=[ + 'psutil~=5.0', + 'portalocker~=1.0', + ], tests_require=['pytest'], ) diff --git a/tests/lock_acquire.py b/tests/lock_acquire.py new file mode 100644 index 0000000..c1b5b21 --- /dev/null +++ b/tests/lock_acquire.py @@ -0,0 +1,28 @@ +import sys +import os +import time +import datetime +from msal_extensions import CrossPlatLock + + +def main(hold_time): + # type: (datetime.timedelta) -> None + """ + Grabs a lock from a well-known file in order to test the CrossPlatLock class across processes. + :param hold_time: The approximate duration that this process should hold onto the lock. + :return: None + """ + pid = os.getpid() + print('{} starting'.format(pid)) + with CrossPlatLock('./delete_me.lockfile'): + print('{} has acquired the lock'.format(pid)) + time.sleep(hold_time.total_seconds()) + print('{} is releasing the lock'.format(pid)) + print('{} done.'.format(pid)) + + +if __name__ == '__main__': + lock_hold_time = datetime.timedelta(seconds=5) + if len(sys.argv) > 1: + hold_time = datetime.timedelta(seconds=int(sys.argv[1])) + main(lock_hold_time) diff --git a/tests/test_crossplatlock.py b/tests/test_crossplatlock.py new file mode 100644 index 0000000..3ddc22d --- /dev/null +++ b/tests/test_crossplatlock.py @@ -0,0 +1,18 @@ +import pytest +from msal_extensions import CrossPlatLock + + +def test_ensure_file_deleted(): + lockfile = './test_lock_1.txt' + + try: + FileNotFoundError + except NameError: + FileNotFoundError = IOError + + with CrossPlatLock(lockfile): + pass + + with pytest.raises(FileNotFoundError): + with open(lockfile): + pass From 4e547cdc24603999b8db0e23ed4df6ef1c86553f Mon Sep 17 00:00:00 2001 From: Martin Strobel Date: Tue, 14 May 2019 10:07:45 -0700 Subject: [PATCH 2/6] Adding class doc-comment. --- msal_extensions/_cache_lock.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/msal_extensions/_cache_lock.py b/msal_extensions/_cache_lock.py index ab85a87..0c23ce0 100644 --- a/msal_extensions/_cache_lock.py +++ b/msal_extensions/_cache_lock.py @@ -6,6 +6,10 @@ class CrossPlatLock(object): + """ + Offers a mechanism for waiting until another process is finished interacting with a shared resource. This is + specifically written to interact with a class of the same name in the .NET extensions library. + """ TIMEOUT = datetime.timedelta(minutes=1) RETRY_WAIT = datetime.timedelta(milliseconds=100) RETRY_COUNT = int(TIMEOUT.total_seconds() / RETRY_WAIT.total_seconds()) From 437e03238d89ad9659bb4f958cc06001ebd59c73 Mon Sep 17 00:00:00 2001 From: Martin Strobel Date: Wed, 15 May 2019 09:28:27 -0700 Subject: [PATCH 3/6] Remove dependency on psutil --- msal_extensions/_cache_lock.py | 5 ++--- setup.py | 1 - 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/msal_extensions/_cache_lock.py b/msal_extensions/_cache_lock.py index 0c23ce0..0c12206 100644 --- a/msal_extensions/_cache_lock.py +++ b/msal_extensions/_cache_lock.py @@ -1,7 +1,7 @@ import os +import sys import errno import datetime -import psutil import portalocker @@ -19,14 +19,13 @@ def __init__(self, lockfile_path): def __enter__(self): pid = os.getpid() - proc = psutil.Process(pid) lock_dir = os.path.dirname(self._lockpath) if not os.path.exists(lock_dir): os.makedirs(lock_dir) self._fh = open(self._lockpath, 'wb+', buffering=0) portalocker.lock(self._fh, portalocker.LOCK_EX) - self._fh.write('{} {}'.format(pid, proc.name()).encode('utf-8')) + self._fh.write('{} {}'.format(pid, sys.argv[0]).encode('utf-8')) def __exit__(self, *args): self._fh.close() diff --git a/setup.py b/setup.py index 3542ae1..cbfe080 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,6 @@ 'Development Status :: 2 - Pre-Alpha', ], install_requires=[ - 'psutil~=5.0', 'portalocker~=1.0', ], tests_require=['pytest'], From bda687b818f4b474009932566a83d06970c458a1 Mon Sep 17 00:00:00 2001 From: Martin Strobel Date: Wed, 15 May 2019 09:36:01 -0700 Subject: [PATCH 4/6] Fixing naming convention --- msal_extensions/__init__.py | 2 -- msal_extensions/{_cache_lock.py => cache_lock.py} | 0 tests/test_crossplatlock.py | 2 +- 3 files changed, 1 insertion(+), 3 deletions(-) rename msal_extensions/{_cache_lock.py => cache_lock.py} (100%) diff --git a/msal_extensions/__init__.py b/msal_extensions/__init__.py index 01da028..f102a9c 100644 --- a/msal_extensions/__init__.py +++ b/msal_extensions/__init__.py @@ -1,3 +1 @@ __version__ = "0.0.1" - -from ._cache_lock import CrossPlatLock diff --git a/msal_extensions/_cache_lock.py b/msal_extensions/cache_lock.py similarity index 100% rename from msal_extensions/_cache_lock.py rename to msal_extensions/cache_lock.py diff --git a/tests/test_crossplatlock.py b/tests/test_crossplatlock.py index 3ddc22d..ea3c9d5 100644 --- a/tests/test_crossplatlock.py +++ b/tests/test_crossplatlock.py @@ -1,5 +1,5 @@ import pytest -from msal_extensions import CrossPlatLock +from msal_extensions.cache_lock import CrossPlatLock def test_ensure_file_deleted(): From aacee74430c6e3961eef8e889f2acdd63514ecea Mon Sep 17 00:00:00 2001 From: Martin Strobel Date: Wed, 15 May 2019 14:58:05 -0700 Subject: [PATCH 5/6] Removing unused constants --- msal_extensions/cache_lock.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/msal_extensions/cache_lock.py b/msal_extensions/cache_lock.py index 0c12206..4a8063c 100644 --- a/msal_extensions/cache_lock.py +++ b/msal_extensions/cache_lock.py @@ -1,7 +1,6 @@ import os import sys import errno -import datetime import portalocker @@ -10,10 +9,6 @@ class CrossPlatLock(object): Offers a mechanism for waiting until another process is finished interacting with a shared resource. This is specifically written to interact with a class of the same name in the .NET extensions library. """ - TIMEOUT = datetime.timedelta(minutes=1) - RETRY_WAIT = datetime.timedelta(milliseconds=100) - RETRY_COUNT = int(TIMEOUT.total_seconds() / RETRY_WAIT.total_seconds()) - def __init__(self, lockfile_path): self._lockpath = lockfile_path From 1731a6ab48e0db1d0c2dae0ba8ad6a821b4d14fa Mon Sep 17 00:00:00 2001 From: Martin Strobel Date: Wed, 15 May 2019 14:59:02 -0700 Subject: [PATCH 6/6] Removing dir presence assertion --- msal_extensions/cache_lock.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/msal_extensions/cache_lock.py b/msal_extensions/cache_lock.py index 4a8063c..9ef90fd 100644 --- a/msal_extensions/cache_lock.py +++ b/msal_extensions/cache_lock.py @@ -14,9 +14,6 @@ def __init__(self, lockfile_path): def __enter__(self): pid = os.getpid() - lock_dir = os.path.dirname(self._lockpath) - if not os.path.exists(lock_dir): - os.makedirs(lock_dir) self._fh = open(self._lockpath, 'wb+', buffering=0) portalocker.lock(self._fh, portalocker.LOCK_EX)