diff --git a/Doc/library/ipaddress.rst b/Doc/library/ipaddress.rst index 140401d2f36739..1dfa2ec073313b 100644 --- a/Doc/library/ipaddress.rst +++ b/Doc/library/ipaddress.rst @@ -168,18 +168,53 @@ write code that handles both IP versions correctly. Address objects are .. attribute:: is_private - ``True`` if the address is allocated for private networks. See + ``True`` if the address is defined as not globally reachable by iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_ - (for IPv6). + (for IPv6) with the following exceptions: + + * ``is_private`` is ``False`` for the shared address space (``100.64.0.0/10``) + * For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the + semantics of the underlying IPv4 addresses and the following condition holds + (see :attr:`IPv6Address.ipv4_mapped`):: + + address.is_private == address.ipv4_mapped.is_private + + ``is_private`` has value opposite to :attr:`is_global`, except for the shared address space + (``100.64.0.0/10`` range) where they are both ``False``. + + .. versionchanged:: 3.8.20 + + Fixed some false positives and false negatives. + + * ``192.0.0.0/24`` is considered private with the exception of ``192.0.0.9/32`` and + ``192.0.0.10/32`` (previously: only the ``192.0.0.0/29`` sub-range was considered private). + * ``64:ff9b:1::/48`` is considered private. + * ``2002::/16`` is considered private. + * There are exceptions within ``2001::/23`` (otherwise considered private): ``2001:1::1/128``, + ``2001:1::2/128``, ``2001:3::/32``, ``2001:4:112::/48``, ``2001:20::/28``, ``2001:30::/28``. + The exceptions are not considered private. .. attribute:: is_global - ``True`` if the address is allocated for public networks. See + ``True`` if the address is defined as globally reachable by iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_ - (for IPv6). + (for IPv6) with the following exception: + + For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the + semantics of the underlying IPv4 addresses and the following condition holds + (see :attr:`IPv6Address.ipv4_mapped`):: + + address.is_global == address.ipv4_mapped.is_global + + ``is_global`` has value opposite to :attr:`is_private`, except for the shared address space + (``100.64.0.0/10`` range) where they are both ``False``. .. versionadded:: 3.4 + .. versionchanged:: 3.8.20 + + Fixed some false positives and false negatives, see :attr:`is_private` for details. + .. attribute:: is_unspecified ``True`` if the address is unspecified. See :RFC:`5735` (for IPv4) diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index 04491742958967..d2a732298539d9 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -273,7 +273,7 @@ Methods and properties .. testsetup:: - from pathlib import PurePosixPath, PureWindowsPath + from pathlib import PurePath, PurePosixPath, PureWindowsPath Pure paths provide the following methods and properties: @@ -462,6 +462,19 @@ Pure paths provide the following methods and properties: True +.. method:: PurePath.is_relative_to(*other) + + Return whether or not this path is relative to the *other* path. + + >>> p = PurePath('/etc/passwd') + >>> p.is_relative_to('/etc') + True + >>> p.is_relative_to('/usr') + False + + .. versionadded:: 3.9 + + .. method:: PurePath.is_reserved() With :class:`PureWindowsPath`, return ``True`` if the path is considered diff --git a/Doc/library/tempfile.rst b/Doc/library/tempfile.rst index 00acf4b1792375..f182c0feaa7e7c 100644 --- a/Doc/library/tempfile.rst +++ b/Doc/library/tempfile.rst @@ -105,12 +105,12 @@ The module defines the following user-callable items: the truncate method now accepts a ``size`` argument. -.. function:: TemporaryDirectory(suffix=None, prefix=None, dir=None) +.. function:: TemporaryDirectory(suffix=None, prefix=None, dir=None, ignore_cleanup_errors=False) This function securely creates a temporary directory using the same rules as :func:`mkdtemp`. The resulting object can be used as a context manager (see :ref:`tempfile-examples`). On completion of the context or destruction - of the temporary directory object the newly created temporary directory + of the temporary directory object, the newly created temporary directory and all its contents are removed from the filesystem. The directory name can be retrieved from the :attr:`name` attribute of the @@ -119,10 +119,19 @@ The module defines the following user-callable items: the :keyword:`with` statement, if there is one. The directory can be explicitly cleaned up by calling the - :func:`cleanup` method. + :func:`cleanup` method. If *ignore_cleanup_errors* is true, any unhandled + exceptions during explicit or implicit cleanup (such as a + :exc:`PermissionError` removing open files on Windows) will be ignored, + and the remaining removable items deleted on a "best-effort" basis. + Otherwise, errors will be raised in whatever context cleanup occurs + (the :func:`cleanup` call, exiting the context manager, when the object + is garbage-collected or during interpreter shutdown). .. versionadded:: 3.2 + .. versionchanged:: 3.10 + Added *ignore_cleanup_errors* parameter. + .. function:: mkstemp(suffix=None, prefix=None, dir=None, text=False) diff --git a/Doc/tools/susp-ignored.csv b/Doc/tools/susp-ignored.csv index a3c7cf6c8a2d6b..567ac98e3df3e2 100644 --- a/Doc/tools/susp-ignored.csv +++ b/Doc/tools/susp-ignored.csv @@ -161,6 +161,14 @@ library/ipaddress,,:db00,2001:db00::0/24 library/ipaddress,,::,2001:db00::0/24 library/ipaddress,,:db00,2001:db00::0/ffff:ff00:: library/ipaddress,,::,2001:db00::0/ffff:ff00:: +library/ipaddress,,:ff9b,64:ff9b:1::/48 +library/ipaddress,,::,64:ff9b:1::/48 +library/ipaddress,,::,2001:: +library/ipaddress,,::,2001:1:: +library/ipaddress,,::,2001:3:: +library/ipaddress,,::,2001:4:112:: +library/ipaddress,,::,2001:20:: +library/ipaddress,,::,2001:30:: library/itertools,,:step,elements from seq[start:stop:step] library/itertools,,:stop,elements from seq[start:stop:step] library/logging.handlers,,:port,host:port diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index 686abb378b930c..09f74f2788849c 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -2650,3 +2650,11 @@ post-handshake TLS encrypted data. Security issue reported as `_ by Aapo Oksman. Patch by Gregory P. Smith. +Notable changes in 3.7.17.4 +========================= + +ipaddress +--------- + +* Fixed ``is_global`` and ``is_private`` behavior in ``IPv4Address``, + ``IPv6Address``, ``IPv4Network`` and ``IPv6Network``. diff --git a/Lib/ipaddress.py b/Lib/ipaddress.py index 54882934c3dc1d..7d1bbdcbc0819e 100644 --- a/Lib/ipaddress.py +++ b/Lib/ipaddress.py @@ -1345,18 +1345,41 @@ def is_reserved(self): @property @functools.lru_cache() def is_private(self): - """Test if this address is allocated for private networks. + """``True`` if the address is defined as not globally reachable by + iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_ + (for IPv6) with the following exceptions: - Returns: - A boolean, True if the address is reserved per - iana-ipv4-special-registry. + * ``is_private`` is ``False`` for ``100.64.0.0/10`` + * For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the + semantics of the underlying IPv4 addresses and the following condition holds + (see :attr:`IPv6Address.ipv4_mapped`):: + + address.is_private == address.ipv4_mapped.is_private + ``is_private`` has value opposite to :attr:`is_global`, except for the ``100.64.0.0/10`` + IPv4 range where they are both ``False``. """ - return any(self in net for net in self._constants._private_networks) + return ( + any(self in net for net in self._constants._private_networks) + and all(self not in net for net in self._constants._private_networks_exceptions) + ) @property @functools.lru_cache() def is_global(self): + """``True`` if the address is defined as globally reachable by + iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_ + (for IPv6) with the following exception: + + For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the + semantics of the underlying IPv4 addresses and the following condition holds + (see :attr:`IPv6Address.ipv4_mapped`):: + + address.is_global == address.ipv4_mapped.is_global + + ``is_global`` has value opposite to :attr:`is_private`, except for the ``100.64.0.0/10`` + IPv4 range where they are both ``False``. + """ return self not in self._constants._public_network and not self.is_private @property @@ -1557,13 +1580,15 @@ class _IPv4Constants: _public_network = IPv4Network('100.64.0.0/10') + # Not globally reachable address blocks listed on + # https://www.iana.org/assignments/iana-ipv4-special-registry/iana-ipv4-special-registry.xhtml _private_networks = [ IPv4Network('0.0.0.0/8'), IPv4Network('10.0.0.0/8'), IPv4Network('127.0.0.0/8'), IPv4Network('169.254.0.0/16'), IPv4Network('172.16.0.0/12'), - IPv4Network('192.0.0.0/29'), + IPv4Network('192.0.0.0/24'), IPv4Network('192.0.0.170/31'), IPv4Network('192.0.2.0/24'), IPv4Network('192.168.0.0/16'), @@ -1574,6 +1599,11 @@ class _IPv4Constants: IPv4Network('255.255.255.255/32'), ] + _private_networks_exceptions = [ + IPv4Network('192.0.0.9/32'), + IPv4Network('192.0.0.10/32'), + ] + _reserved_network = IPv4Network('240.0.0.0/4') _unspecified_address = IPv4Address('0.0.0.0') @@ -1964,23 +1994,42 @@ def is_site_local(self): @property @functools.lru_cache() def is_private(self): - """Test if this address is allocated for private networks. + """``True`` if the address is defined as not globally reachable by + iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_ + (for IPv6) with the following exceptions: - Returns: - A boolean, True if the address is reserved per - iana-ipv6-special-registry. + * ``is_private`` is ``False`` for ``100.64.0.0/10`` + * For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the + semantics of the underlying IPv4 addresses and the following condition holds + (see :attr:`IPv6Address.ipv4_mapped`):: + + address.is_private == address.ipv4_mapped.is_private + ``is_private`` has value opposite to :attr:`is_global`, except for the ``100.64.0.0/10`` + IPv4 range where they are both ``False``. """ - return any(self in net for net in self._constants._private_networks) + ipv4_mapped = self.ipv4_mapped + if ipv4_mapped is not None: + return ipv4_mapped.is_private + return ( + any(self in net for net in self._constants._private_networks) + and all(self not in net for net in self._constants._private_networks_exceptions) + ) @property def is_global(self): - """Test if this address is allocated for public networks. + """``True`` if the address is defined as globally reachable by + iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_ + (for IPv6) with the following exception: - Returns: - A boolean, true if the address is not reserved per - iana-ipv6-special-registry. + For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the + semantics of the underlying IPv4 addresses and the following condition holds + (see :attr:`IPv6Address.ipv4_mapped`):: + + address.is_global == address.ipv4_mapped.is_global + ``is_global`` has value opposite to :attr:`is_private`, except for the ``100.64.0.0/10`` + IPv4 range where they are both ``False``. """ return not self.is_private @@ -2217,19 +2266,31 @@ class _IPv6Constants: _multicast_network = IPv6Network('ff00::/8') + # Not globally reachable address blocks listed on + # https://www.iana.org/assignments/iana-ipv6-special-registry/iana-ipv6-special-registry.xhtml _private_networks = [ IPv6Network('::1/128'), IPv6Network('::/128'), IPv6Network('::ffff:0:0/96'), + IPv6Network('64:ff9b:1::/48'), IPv6Network('100::/64'), IPv6Network('2001::/23'), - IPv6Network('2001:2::/48'), IPv6Network('2001:db8::/32'), - IPv6Network('2001:10::/28'), + # IANA says N/A, let's consider it not globally reachable to be safe + IPv6Network('2002::/16'), IPv6Network('fc00::/7'), IPv6Network('fe80::/10'), ] + _private_networks_exceptions = [ + IPv6Network('2001:1::1/128'), + IPv6Network('2001:1::2/128'), + IPv6Network('2001:3::/32'), + IPv6Network('2001:4:112::/48'), + IPv6Network('2001:20::/28'), + IPv6Network('2001:30::/28'), + ] + _reserved_networks = [ IPv6Network('::/8'), IPv6Network('100::/8'), IPv6Network('200::/7'), IPv6Network('400::/6'), diff --git a/Lib/pathlib.py b/Lib/pathlib.py index f2967492ae8463..84e18528586d20 100644 --- a/Lib/pathlib.py +++ b/Lib/pathlib.py @@ -901,6 +901,15 @@ def relative_to(self, *other): return self._from_parsed_parts('', root if n == 1 else '', abs_parts[n:]) + def is_relative_to(self, *other): + """Return True if the path is relative to another path or False. + """ + try: + self.relative_to(*other) + return True + except ValueError: + return False + @property def parts(self): """An object providing sequence-like access to the diff --git a/Lib/platform.py b/Lib/platform.py index 7af46ffd17728a..7bf5a6b381ce77 100755 --- a/Lib/platform.py +++ b/Lib/platform.py @@ -1282,7 +1282,7 @@ def python_version_tuple(): will always include the patchlevel (it defaults to 0). """ - return tuple(_sys_version()[1].split('.')) + return tuple(_sys_version()[1].split('.', 2)) def python_branch(): diff --git a/Lib/site.py b/Lib/site.py index 878658827ca5c4..d1e55610ddcee4 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -375,6 +375,7 @@ def setcopyright(): "Jython is maintained by the Jython developers (www.jython.org).") else: builtins.credits = _sitebuiltins._Printer("credits", """\ + ActivePython is a Python distribution by ActiveState Software Inc. Thanks to CWI, CNRI, BeOpen.com, Zope Corporation and a cast of thousands for supporting Python development. See www.python.org for more information.""") files, dirs = [], [] @@ -382,11 +383,19 @@ def setcopyright(): # PEP 420 for more details. if hasattr(os, '__file__'): here = os.path.dirname(os.__file__) - files.extend(["LICENSE.txt", "LICENSE"]) - dirs.extend([os.path.join(here, os.pardir), here, os.curdir]) + files.extend(["LICENSE.txt", "LICENSE", "License.txt"]) + dirs.extend( + [os.path.join(here, os.pardir, "Doc"), # dev build and installation on Windows + os.path.join(here, os.pardir, os.pardir, "doc", # APy install on Linux + "python%s.%s" % sys.version_info[:2]), + # APy install on Mac OS X + os.path.join(here, os.pardir, os.pardir, "Resources", + "Python.app", "Contents", "Resources", "English.lproj", + "Help"), + here]) # dev build on Linux builtins.license = _sitebuiltins._Printer( "license", - "See https://www.python.org/psf/license/", + "See https://www.activestate.com/activepython/license-agreement", files, dirs) diff --git a/Lib/tempfile.py b/Lib/tempfile.py index f92db5313215e9..3cb53a815b4b11 100644 --- a/Lib/tempfile.py +++ b/Lib/tempfile.py @@ -774,7 +774,7 @@ def writelines(self, iterable): return rv -class TemporaryDirectory(object): +class TemporaryDirectory: """Create and return a temporary directory. This has the same behavior as mkdtemp but can be used as a context manager. For example: @@ -786,15 +786,49 @@ class TemporaryDirectory(object): in it are removed. """ - def __init__(self, suffix=None, prefix=None, dir=None): + def __init__(self, suffix=None, prefix=None, dir=None, + ignore_cleanup_errors=False): self.name = mkdtemp(suffix, prefix, dir) + self._ignore_cleanup_errors = ignore_cleanup_errors self._finalizer = _weakref.finalize( self, self._cleanup, self.name, - warn_message="Implicitly cleaning up {!r}".format(self)) + warn_message="Implicitly cleaning up {!r}".format(self), + ignore_errors=self._ignore_cleanup_errors) @classmethod - def _cleanup(cls, name, warn_message): - _shutil.rmtree(name) + def _rmtree(cls, name, ignore_errors=False): + def onerror(func, path, exc_info): + if issubclass(exc_info[0], PermissionError): + def resetperms(path): + try: + _os.chflags(path, 0) + except AttributeError: + pass + _os.chmod(path, 0o700) + + try: + if path != name: + resetperms(_os.path.dirname(path)) + resetperms(path) + + try: + _os.unlink(path) + # PermissionError is raised on FreeBSD for directories + except (IsADirectoryError, PermissionError): + cls._rmtree(path, ignore_errors=ignore_errors) + except FileNotFoundError: + pass + elif issubclass(exc_info[0], FileNotFoundError): + pass + else: + if not ignore_errors: + raise + + _shutil.rmtree(name, onerror=onerror) + + @classmethod + def _cleanup(cls, name, warn_message, ignore_errors=False): + cls._rmtree(name, ignore_errors=ignore_errors) _warnings.warn(warn_message, ResourceWarning) def __repr__(self): @@ -807,5 +841,5 @@ def __exit__(self, exc, value, tb): self.cleanup() def cleanup(self): - if self._finalizer.detach(): - _shutil.rmtree(self.name) + if self._finalizer.detach() or _os.path.exists(self.name): + self._rmtree(self.name, ignore_errors=self._ignore_cleanup_errors) diff --git a/Lib/test/test_ipaddress.py b/Lib/test/test_ipaddress.py index 1fb6a929dc2d9e..65b9168fe0ead7 100644 --- a/Lib/test/test_ipaddress.py +++ b/Lib/test/test_ipaddress.py @@ -1766,6 +1766,10 @@ def testReservedIpv4(self): self.assertEqual(True, ipaddress.ip_address( '172.31.255.255').is_private) self.assertEqual(False, ipaddress.ip_address('172.32.0.0').is_private) + self.assertFalse(ipaddress.ip_address('192.0.0.0').is_global) + self.assertTrue(ipaddress.ip_address('192.0.0.9').is_global) + self.assertTrue(ipaddress.ip_address('192.0.0.10').is_global) + self.assertFalse(ipaddress.ip_address('192.0.0.255').is_global) self.assertEqual(True, ipaddress.ip_address('169.254.100.200').is_link_local) @@ -1781,6 +1785,40 @@ def testReservedIpv4(self): self.assertEqual(False, ipaddress.ip_address('128.0.0.0').is_loopback) self.assertEqual(True, ipaddress.ip_network('0.0.0.0').is_unspecified) + def testPrivateNetworks(self): + self.assertEqual(False, ipaddress.ip_network("0.0.0.0/0").is_private) + self.assertEqual(False, ipaddress.ip_network("1.0.0.0/8").is_private) + + self.assertEqual(True, ipaddress.ip_network("0.0.0.0/8").is_private) + self.assertEqual(True, ipaddress.ip_network("10.0.0.0/8").is_private) + self.assertEqual(True, ipaddress.ip_network("127.0.0.0/8").is_private) + self.assertEqual(True, ipaddress.ip_network("169.254.0.0/16").is_private) + self.assertEqual(True, ipaddress.ip_network("172.16.0.0/12").is_private) + self.assertEqual(True, ipaddress.ip_network("192.0.0.0/29").is_private) + self.assertEqual(False, ipaddress.ip_network("192.0.0.9/32").is_private) + self.assertEqual(True, ipaddress.ip_network("192.0.0.170/31").is_private) + self.assertEqual(True, ipaddress.ip_network("192.0.2.0/24").is_private) + self.assertEqual(True, ipaddress.ip_network("192.168.0.0/16").is_private) + self.assertEqual(True, ipaddress.ip_network("198.18.0.0/15").is_private) + self.assertEqual(True, ipaddress.ip_network("198.51.100.0/24").is_private) + self.assertEqual(True, ipaddress.ip_network("203.0.113.0/24").is_private) + self.assertEqual(True, ipaddress.ip_network("240.0.0.0/4").is_private) + self.assertEqual(True, ipaddress.ip_network("255.255.255.255/32").is_private) + + self.assertEqual(False, ipaddress.ip_network("::/0").is_private) + self.assertEqual(False, ipaddress.ip_network("::ff/128").is_private) + + self.assertEqual(True, ipaddress.ip_network("::1/128").is_private) + self.assertEqual(True, ipaddress.ip_network("::/128").is_private) + self.assertEqual(True, ipaddress.ip_network("::ffff:0:0/96").is_private) + self.assertEqual(True, ipaddress.ip_network("100::/64").is_private) + self.assertEqual(True, ipaddress.ip_network("2001:2::/48").is_private) + self.assertEqual(False, ipaddress.ip_network("2001:3::/48").is_private) + self.assertEqual(True, ipaddress.ip_network("2001:db8::/32").is_private) + self.assertEqual(True, ipaddress.ip_network("2001:10::/28").is_private) + self.assertEqual(True, ipaddress.ip_network("fc00::/7").is_private) + self.assertEqual(True, ipaddress.ip_network("fe80::/10").is_private) + def testReservedIpv6(self): self.assertEqual(True, ipaddress.ip_network('ffff::').is_multicast) @@ -1854,6 +1892,20 @@ def testReservedIpv6(self): self.assertEqual(True, ipaddress.ip_address('0::0').is_unspecified) self.assertEqual(False, ipaddress.ip_address('::1').is_unspecified) + self.assertFalse(ipaddress.ip_address('64:ff9b:1::').is_global) + self.assertFalse(ipaddress.ip_address('2001::').is_global) + self.assertTrue(ipaddress.ip_address('2001:1::1').is_global) + self.assertTrue(ipaddress.ip_address('2001:1::2').is_global) + self.assertFalse(ipaddress.ip_address('2001:2::').is_global) + self.assertTrue(ipaddress.ip_address('2001:3::').is_global) + self.assertFalse(ipaddress.ip_address('2001:4::').is_global) + self.assertTrue(ipaddress.ip_address('2001:4:112::').is_global) + self.assertFalse(ipaddress.ip_address('2001:10::').is_global) + self.assertTrue(ipaddress.ip_address('2001:20::').is_global) + self.assertTrue(ipaddress.ip_address('2001:30::').is_global) + self.assertFalse(ipaddress.ip_address('2001:40::').is_global) + self.assertFalse(ipaddress.ip_address('2002::').is_global) + # some generic IETF reserved addresses self.assertEqual(True, ipaddress.ip_address('100::').is_reserved) self.assertEqual(True, ipaddress.ip_network('4000::1/128').is_reserved) diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index 83181b4322e922..fb4b4b49eff145 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -616,6 +616,40 @@ def test_relative_to_common(self): self.assertRaises(ValueError, p.relative_to, '') self.assertRaises(ValueError, p.relative_to, P('a')) + def test_is_relative_to_common(self): + P = self.cls + p = P('a/b') + self.assertRaises(TypeError, p.is_relative_to) + self.assertRaises(TypeError, p.is_relative_to, b'a') + self.assertTrue(p.is_relative_to(P())) + self.assertTrue(p.is_relative_to('')) + self.assertTrue(p.is_relative_to(P('a'))) + self.assertTrue(p.is_relative_to('a/')) + self.assertTrue(p.is_relative_to(P('a/b'))) + self.assertTrue(p.is_relative_to('a/b')) + # With several args. + self.assertTrue(p.is_relative_to('a', 'b')) + # Unrelated paths. + self.assertFalse(p.is_relative_to(P('c'))) + self.assertFalse(p.is_relative_to(P('a/b/c'))) + self.assertFalse(p.is_relative_to(P('a/c'))) + self.assertFalse(p.is_relative_to(P('/a'))) + p = P('/a/b') + self.assertTrue(p.is_relative_to(P('/'))) + self.assertTrue(p.is_relative_to('/')) + self.assertTrue(p.is_relative_to(P('/a'))) + self.assertTrue(p.is_relative_to('/a')) + self.assertTrue(p.is_relative_to('/a/')) + self.assertTrue(p.is_relative_to(P('/a/b'))) + self.assertTrue(p.is_relative_to('/a/b')) + # Unrelated paths. + self.assertFalse(p.is_relative_to(P('/c'))) + self.assertFalse(p.is_relative_to(P('/a/b/c'))) + self.assertFalse(p.is_relative_to(P('/a/c'))) + self.assertFalse(p.is_relative_to(P())) + self.assertFalse(p.is_relative_to('')) + self.assertFalse(p.is_relative_to(P('a'))) + def test_pickling_common(self): P = self.cls p = P('/a/b') @@ -1059,6 +1093,59 @@ def test_relative_to(self): self.assertRaises(ValueError, p.relative_to, P('//z/Share/Foo')) self.assertRaises(ValueError, p.relative_to, P('//Server/z/Foo')) + def test_is_relative_to(self): + P = self.cls + p = P('C:Foo/Bar') + self.assertTrue(p.is_relative_to(P('c:'))) + self.assertTrue(p.is_relative_to('c:')) + self.assertTrue(p.is_relative_to(P('c:foO'))) + self.assertTrue(p.is_relative_to('c:foO')) + self.assertTrue(p.is_relative_to('c:foO/')) + self.assertTrue(p.is_relative_to(P('c:foO/baR'))) + self.assertTrue(p.is_relative_to('c:foO/baR')) + # Unrelated paths. + self.assertFalse(p.is_relative_to(P())) + self.assertFalse(p.is_relative_to('')) + self.assertFalse(p.is_relative_to(P('d:'))) + self.assertFalse(p.is_relative_to(P('/'))) + self.assertFalse(p.is_relative_to(P('Foo'))) + self.assertFalse(p.is_relative_to(P('/Foo'))) + self.assertFalse(p.is_relative_to(P('C:/Foo'))) + self.assertFalse(p.is_relative_to(P('C:Foo/Bar/Baz'))) + self.assertFalse(p.is_relative_to(P('C:Foo/Baz'))) + p = P('C:/Foo/Bar') + self.assertTrue(p.is_relative_to('c:')) + self.assertTrue(p.is_relative_to(P('c:/'))) + self.assertTrue(p.is_relative_to(P('c:/foO'))) + self.assertTrue(p.is_relative_to('c:/foO/')) + self.assertTrue(p.is_relative_to(P('c:/foO/baR'))) + self.assertTrue(p.is_relative_to('c:/foO/baR')) + # Unrelated paths. + self.assertFalse(p.is_relative_to(P('C:/Baz'))) + self.assertFalse(p.is_relative_to(P('C:/Foo/Bar/Baz'))) + self.assertFalse(p.is_relative_to(P('C:/Foo/Baz'))) + self.assertFalse(p.is_relative_to(P('C:Foo'))) + self.assertFalse(p.is_relative_to(P('d:'))) + self.assertFalse(p.is_relative_to(P('d:/'))) + self.assertFalse(p.is_relative_to(P('/'))) + self.assertFalse(p.is_relative_to(P('/Foo'))) + self.assertFalse(p.is_relative_to(P('//C/Foo'))) + # UNC paths. + p = P('//Server/Share/Foo/Bar') + self.assertTrue(p.is_relative_to(P('//sErver/sHare'))) + self.assertTrue(p.is_relative_to('//sErver/sHare')) + self.assertTrue(p.is_relative_to('//sErver/sHare/')) + self.assertTrue(p.is_relative_to(P('//sErver/sHare/Foo'))) + self.assertTrue(p.is_relative_to('//sErver/sHare/Foo')) + self.assertTrue(p.is_relative_to('//sErver/sHare/Foo/')) + self.assertTrue(p.is_relative_to(P('//sErver/sHare/Foo/Bar'))) + self.assertTrue(p.is_relative_to('//sErver/sHare/Foo/Bar')) + # Unrelated paths. + self.assertFalse(p.is_relative_to(P('/Server/Share/Foo'))) + self.assertFalse(p.is_relative_to(P('c:/Server/Share/Foo'))) + self.assertFalse(p.is_relative_to(P('//z/Share/Foo'))) + self.assertFalse(p.is_relative_to(P('//Server/z/Foo'))) + def test_is_absolute(self): P = self.cls # Under NT, only paths with both a drive and a root are absolute diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py index 2d47e70bd03288..8171aa6e0ae52a 100644 --- a/Lib/test/test_tempfile.py +++ b/Lib/test/test_tempfile.py @@ -1312,13 +1312,17 @@ def __exit__(self, *exc_info): d.clear() d.update(c) + class TestTemporaryDirectory(BaseTestCase): """Test TemporaryDirectory().""" - def do_create(self, dir=None, pre="", suf="", recurse=1): + def do_create(self, dir=None, pre="", suf="", recurse=1, dirs=1, files=1, + ignore_cleanup_errors=False): if dir is None: dir = tempfile.gettempdir() - tmp = tempfile.TemporaryDirectory(dir=dir, prefix=pre, suffix=suf) + tmp = tempfile.TemporaryDirectory( + dir=dir, prefix=pre, suffix=suf, + ignore_cleanup_errors=ignore_cleanup_errors) self.nameCheck(tmp.name, dir, pre, suf) # Create a subdirectory and some files if recurse: @@ -1351,7 +1355,31 @@ def test_explicit_cleanup(self): finally: os.rmdir(dir) - @support.skip_unless_symlink + def test_explict_cleanup_ignore_errors(self): + """Test that cleanup doesn't return an error when ignoring them.""" + with tempfile.TemporaryDirectory() as working_dir: + temp_dir = self.do_create( + dir=working_dir, ignore_cleanup_errors=True) + temp_path = pathlib.Path(temp_dir.name) + self.assertTrue(temp_path.exists(), + f"TemporaryDirectory {temp_path!s} does not exist") + with open(temp_path / "a_file.txt", "w+t") as open_file: + open_file.write("Hello world!\n") + temp_dir.cleanup() + self.assertEqual(len(list(temp_path.glob("*"))), + int(sys.platform.startswith("win")), + "Unexpected number of files in " + f"TemporaryDirectory {temp_path!s}") + self.assertEqual( + temp_path.exists(), + sys.platform.startswith("win"), + f"TemporaryDirectory {temp_path!s} existance state unexpected") + temp_dir.cleanup() + self.assertFalse( + temp_path.exists(), + f"TemporaryDirectory {temp_path!s} exists after cleanup") + + @os_helper.skip_unless_symlink def test_cleanup_with_symlink_to_a_directory(self): # cleanup() should not follow symlinks to directories (issue #12464) d1 = self.do_create() @@ -1385,6 +1413,27 @@ def test_del_on_collection(self): finally: os.rmdir(dir) + @support.cpython_only + def test_del_on_collection_ignore_errors(self): + """Test that ignoring errors works when TemporaryDirectory is gced.""" + with tempfile.TemporaryDirectory() as working_dir: + temp_dir = self.do_create( + dir=working_dir, ignore_cleanup_errors=True) + temp_path = pathlib.Path(temp_dir.name) + self.assertTrue(temp_path.exists(), + f"TemporaryDirectory {temp_path!s} does not exist") + with open(temp_path / "a_file.txt", "w+t") as open_file: + open_file.write("Hello world!\n") + del temp_dir + self.assertEqual(len(list(temp_path.glob("*"))), + int(sys.platform.startswith("win")), + "Unexpected number of files in " + f"TemporaryDirectory {temp_path!s}") + self.assertEqual( + temp_path.exists(), + sys.platform.startswith("win"), + f"TemporaryDirectory {temp_path!s} existance state unexpected") + def test_del_on_shutdown(self): # A TemporaryDirectory may be cleaned up during shutdown with self.do_create() as dir: @@ -1417,6 +1466,43 @@ def test_del_on_shutdown(self): self.assertNotIn("Exception ", err) self.assertIn("ResourceWarning: Implicitly cleaning up", err) + def test_del_on_shutdown_ignore_errors(self): + """Test ignoring errors works when a tempdir is gc'ed on shutdown.""" + with tempfile.TemporaryDirectory() as working_dir: + code = """if True: + import pathlib + import sys + import tempfile + import warnings + + temp_dir = tempfile.TemporaryDirectory( + dir={working_dir!r}, ignore_cleanup_errors=True) + sys.stdout.buffer.write(temp_dir.name.encode()) + + temp_dir_2 = pathlib.Path(temp_dir.name) / "test_dir" + temp_dir_2.mkdir() + with open(temp_dir_2 / "test0.txt", "w") as test_file: + test_file.write("Hello world!") + open_file = open(temp_dir_2 / "open_file.txt", "w") + open_file.write("Hello world!") + + warnings.filterwarnings("always", category=ResourceWarning) + """.format(working_dir=working_dir) + __, out, err = script_helper.assert_python_ok("-c", code) + temp_path = pathlib.Path(out.decode().strip()) + self.assertEqual(len(list(temp_path.glob("*"))), + int(sys.platform.startswith("win")), + "Unexpected number of files in " + f"TemporaryDirectory {temp_path!s}") + self.assertEqual( + temp_path.exists(), + sys.platform.startswith("win"), + f"TemporaryDirectory {temp_path!s} existance state unexpected") + err = err.decode('utf-8', 'backslashreplace') + self.assertNotIn("Exception", err) + self.assertNotIn("Error", err) + self.assertIn("ResourceWarning: Implicitly cleaning up", err) + def test_exit_on_shutdown(self): # Issue #22427 with self.do_create() as dir: diff --git a/Makefile.pre.in b/Makefile.pre.in index ef2bfb1cd29d5c..d71f915dfef7bc 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1406,7 +1406,6 @@ libinstall: build_all $(srcdir)/Modules/xxmodule.c done $(INSTALL_DATA) `cat pybuilddir.txt`/_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH).py \ $(DESTDIR)$(LIBDEST); \ - $(INSTALL_DATA) $(srcdir)/LICENSE $(DESTDIR)$(LIBDEST)/LICENSE.txt if test -d $(DESTDIR)$(LIBDEST)/distutils/tests; then \ $(INSTALL_DATA) $(srcdir)/Modules/xxmodule.c \ $(DESTDIR)$(LIBDEST)/distutils/tests ; \ @@ -1545,7 +1544,7 @@ sharedinstall: sharedmods $(RUNSHARED) $(PYTHON_FOR_BUILD) $(srcdir)/setup.py install \ --prefix=$(prefix) \ --install-scripts=$(BINDIR) \ - --install-platlib=$(DESTSHARED) \ + --install-platlib=${exec_prefix}/lib/python$(VERSION)/lib-dynload \ --root=$(DESTDIR)/ -rm $(DESTDIR)$(DESTSHARED)/_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH).py -rm -r $(DESTDIR)$(DESTSHARED)/__pycache__ diff --git a/Misc/NEWS.d/next/Library/2019-07-27-18-00-43.bpo-37689.glEmZi.rst b/Misc/NEWS.d/next/Library/2019-07-27-18-00-43.bpo-37689.glEmZi.rst new file mode 100644 index 00000000000000..2787f9e849c20b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-07-27-18-00-43.bpo-37689.glEmZi.rst @@ -0,0 +1 @@ +Add :meth:`is_relative_to` in :class:`PurePath` to determine whether or not one path is relative to another. diff --git a/Misc/NEWS.d/next/Library/2021-03-07-23-23-03.bpo-29982.Q9iszT.rst b/Misc/NEWS.d/next/Library/2021-03-07-23-23-03.bpo-29982.Q9iszT.rst new file mode 100644 index 00000000000000..fd71bc6e4e0df7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-03-07-23-23-03.bpo-29982.Q9iszT.rst @@ -0,0 +1,3 @@ +Add optional parameter *ignore_cleanup_errors* to +:func:`tempfile.TemporaryDirectory` and allow multiple :func:`cleanup` attempts. +Contributed by C.A.M. Gerlach. diff --git a/Misc/NEWS.d/next/Library/2024-03-14-01-38-44.gh-issue-113171.VFnObz.rst b/Misc/NEWS.d/next/Library/2024-03-14-01-38-44.gh-issue-113171.VFnObz.rst new file mode 100644 index 00000000000000..f9a72473be4e2c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-03-14-01-38-44.gh-issue-113171.VFnObz.rst @@ -0,0 +1,9 @@ +Fixed various false positives and false negatives in + +* :attr:`ipaddress.IPv4Address.is_private` (see these docs for details) +* :attr:`ipaddress.IPv4Address.is_global` +* :attr:`ipaddress.IPv6Address.is_private` +* :attr:`ipaddress.IPv6Address.is_global` + +Also in the corresponding :class:`ipaddress.IPv4Network` and :class:`ipaddress.IPv6Network` +attributes. diff --git a/Misc/NEWS.d/next/Security/2024-04-19-11-21-13.gh-issue-114572.t1QMQD.rst b/Misc/NEWS.d/next/Security/2024-04-19-11-21-13.gh-issue-114572.t1QMQD.rst new file mode 100644 index 00000000000000..b4f9fe64db0615 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2024-04-19-11-21-13.gh-issue-114572.t1QMQD.rst @@ -0,0 +1,4 @@ +:meth:`ssl.SSLContext.cert_store_stats` and +:meth:`ssl.SSLContext.get_ca_certs` now correctly lock access to the +certificate store, when the :class:`ssl.SSLContext` is shared across +multiple threads. diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 719f8e8ca308d1..1458888d20ce6d 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -167,6 +167,10 @@ static void _PySSLFixErrno(void) { # define PY_OPENSSL_1_1_API 1 #endif +#if (OPENSSL_VERSION_NUMBER >= 0x30300000L) && !defined(LIBRESSL_VERSION_NUMBER) +# define OPENSSL_VERSION_3_3 1 +#endif + /* SNI support (client- and server-side) appeared in OpenSSL 1.0.0 and 0.9.8f * This includes the SSL_set_SSL_CTX() function. */ @@ -207,6 +211,16 @@ static void _PySSLFixErrno(void) { #define HAVE_OPENSSL_CRYPTO_LOCK #endif +/* OpenSSL 1.1+ allows locking X509_STORE, 1.0.2 doesn't. */ +#ifdef OPENSSL_VERSION_1_1 +#define HAVE_OPENSSL_X509_STORE_LOCK +#endif + +/* OpenSSL 3.3 added the X509_STORE_get1_objects API */ +#ifdef OPENSSL_VERSION_3_3 +#define HAVE_OPENSSL_X509_STORE_GET1_OBJECTS 1 +#endif + #if defined(OPENSSL_VERSION_1_1) && !defined(OPENSSL_NO_SSL2) #define OPENSSL_NO_SSL2 #endif @@ -4517,6 +4531,54 @@ set_sni_callback(PySSLContext *self, PyObject *arg, void *c) #endif } +/* Shim of X509_STORE_get1_objects API from OpenSSL 3.3 + * Only available with the X509_STORE_lock() API */ +#if defined(HAVE_OPENSSL_X509_STORE_LOCK) && !defined(OPENSSL_VERSION_3_3) +#define HAVE_OPENSSL_X509_STORE_GET1_OBJECTS 1 + +static X509_OBJECT *x509_object_dup(const X509_OBJECT *obj) +{ + int ok; + X509_OBJECT *ret = X509_OBJECT_new(); + if (ret == NULL) { + return NULL; + } + switch (X509_OBJECT_get_type(obj)) { + case X509_LU_X509: + ok = X509_OBJECT_set1_X509(ret, X509_OBJECT_get0_X509(obj)); + break; + case X509_LU_CRL: + /* X509_OBJECT_get0_X509_CRL was not const-correct prior to 3.0.*/ + ok = X509_OBJECT_set1_X509_CRL( + ret, X509_OBJECT_get0_X509_CRL((X509_OBJECT *)obj)); + break; + default: + /* We cannot duplicate unrecognized types in a polyfill, but it is + * safe to leave an empty object. The caller will ignore it. */ + ok = 1; + break; + } + if (!ok) { + X509_OBJECT_free(ret); + return NULL; + } + return ret; +} + +static STACK_OF(X509_OBJECT) * +X509_STORE_get1_objects(X509_STORE *store) +{ + STACK_OF(X509_OBJECT) *ret; + if (!X509_STORE_lock(store)) { + return NULL; + } + ret = sk_X509_OBJECT_deep_copy(X509_STORE_get0_objects(store), + x509_object_dup, X509_OBJECT_free); + X509_STORE_unlock(store); + return ret; +} +#endif + PyDoc_STRVAR(PySSLContext_sni_callback_doc, "Set a callback that will be called when a server name is provided by the SSL/TLS client in the SNI extension.\n\ \n\ @@ -4546,7 +4608,15 @@ _ssl__SSLContext_cert_store_stats_impl(PySSLContext *self) int x509 = 0, crl = 0, ca = 0, i; store = SSL_CTX_get_cert_store(self->ctx); +#if HAVE_OPENSSL_X509_STORE_GET1_OBJECTS + objs = X509_STORE_get1_objects(store); + if (objs == NULL) { + PyErr_SetString(PyExc_MemoryError, "failed to query cert store"); + return NULL; + } +#else objs = X509_STORE_get0_objects(store); +#endif for (i = 0; i < sk_X509_OBJECT_num(objs); i++) { obj = sk_X509_OBJECT_value(objs, i); switch (X509_OBJECT_get_type(obj)) { @@ -4563,9 +4633,14 @@ _ssl__SSLContext_cert_store_stats_impl(PySSLContext *self) /* Ignore X509_LU_FAIL, X509_LU_RETRY, X509_LU_PKEY. * As far as I can tell they are internal states and never * stored in a cert store */ + /* Ignore unrecognized types. */ break; } } + +#if HAVE_OPENSSL_X509_STORE_GET1_OBJECTS + sk_X509_OBJECT_pop_free(objs, X509_OBJECT_free); +#endif return Py_BuildValue("{sisisi}", "x509", x509, "crl", crl, "x509_ca", ca); } @@ -4597,7 +4672,15 @@ _ssl__SSLContext_get_ca_certs_impl(PySSLContext *self, int binary_form) } store = SSL_CTX_get_cert_store(self->ctx); +#if HAVE_OPENSSL_X509_STORE_GET1_OBJECTS + objs = X509_STORE_get1_objects(store); + if (objs == NULL) { + PyErr_SetString(PyExc_MemoryError, "failed to query cert store"); + return NULL; + } +#else objs = X509_STORE_get0_objects(store); +#endif for (i = 0; i < sk_X509_OBJECT_num(objs); i++) { X509_OBJECT *obj; X509 *cert; @@ -4625,9 +4708,15 @@ _ssl__SSLContext_get_ca_certs_impl(PySSLContext *self, int binary_form) } Py_CLEAR(ci); } +#if HAVE_OPENSSL_X509_STORE_GET1_OBJECTS + sk_X509_OBJECT_pop_free(objs, X509_OBJECT_free); +#endif return rlist; error: +#if HAVE_OPENSSL_X509_STORE_GET1_OBJECTS + sk_X509_OBJECT_pop_free(objs, X509_OBJECT_free); +#endif Py_XDECREF(ci); Py_XDECREF(rlist); return NULL; diff --git a/PC/layout/support/options.py b/PC/layout/support/options.py index 22492f220d606e..e0730446564b5a 100644 --- a/PC/layout/support/options.py +++ b/PC/layout/support/options.py @@ -40,7 +40,6 @@ def public(f): "help": "APPX package", "options": [ "stable", - "pip", "distutils", "tcltk", "idle", @@ -56,7 +55,6 @@ def public(f): "options": [ "dev", "tools", - "pip", "stable", "distutils", "venv", @@ -67,7 +65,6 @@ def public(f): "help": "development kit package", "options": [ "stable", - "pip", "distutils", "tcltk", "idle", diff --git a/PCbuild/_bz2.vcxproj b/PCbuild/_bz2.vcxproj index 3c21848ae28873..581c16f36dccb3 100644 --- a/PCbuild/_bz2.vcxproj +++ b/PCbuild/_bz2.vcxproj @@ -60,23 +60,43 @@ <_ProjectFileVersion>10.0.30319.1 - + + + $(bz2Dir)include;%(AdditionalIncludeDirectories) + WIN32;_FILE_OFFSET_BITS=64;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_DEPRECATE;%(PreprocessorDefinitions) + 4244;4267;%(DisableSpecificWarnings) + + + $(bz2Dir)lib;%(AdditionalLibraryDirectories) + libbz2.lib;%(AdditionalDependencies) + + + $(bz2Dir);%(AdditionalIncludeDirectories) WIN32;_FILE_OFFSET_BITS=64;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_DEPRECATE;%(PreprocessorDefinitions) 4244;4267;%(DisableSpecificWarnings) - - - - - - - - - - + + + + + + + + + + + + + + + + + + + @@ -93,4 +113,4 @@ - \ No newline at end of file + diff --git a/PCbuild/_lzma.vcxproj b/PCbuild/_lzma.vcxproj index d8b159e6a1d853..573f5c53ae38e5 100644 --- a/PCbuild/_lzma.vcxproj +++ b/PCbuild/_lzma.vcxproj @@ -59,7 +59,7 @@ <_ProjectFileVersion>10.0.30319.1 - + $(lzmaDir)src/liblzma/api;%(AdditionalIncludeDirectories) WIN32;_FILE_OFFSET_BITS=64;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_DEPRECATE;LZMA_API_STATIC;%(PreprocessorDefinitions) @@ -68,6 +68,15 @@ $(OutDir)liblzma$(PyDebugExt).lib;%(AdditionalDependencies) + + + $(lzmaDir)include;%(AdditionalIncludeDirectories) + WIN32;_FILE_OFFSET_BITS=64;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_DEPRECATE;LZMA_API_STATIC;%(PreprocessorDefinitions) + + + $(lzmaDir)lib\liblzma.lib;%(AdditionalDependencies) + + @@ -79,6 +88,8 @@ {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} false + + {12728250-16eC-4dc6-94d7-e21dd88947f8} false diff --git a/PCbuild/_sqlite3.vcxproj b/PCbuild/_sqlite3.vcxproj index 1f1a1c8cada19d..97142324738781 100644 --- a/PCbuild/_sqlite3.vcxproj +++ b/PCbuild/_sqlite3.vcxproj @@ -61,10 +61,16 @@ - $(sqlite3Dir);%(AdditionalIncludeDirectories) + $(sqlite3Dir)include;%(AdditionalIncludeDirectories) MODULE_NAME="sqlite3";%(PreprocessorDefinitions) + + + $(sqlite3Dir)lib;%(AdditionalLibraryDirectories) + sqlite3.lib;%(AdditionalDependencies) + + @@ -90,7 +96,7 @@ - + {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} false @@ -103,4 +109,4 @@ - \ No newline at end of file + diff --git a/PCbuild/build_activestate.bat b/PCbuild/build_activestate.bat new file mode 100644 index 00000000000000..766288a3646a19 --- /dev/null +++ b/PCbuild/build_activestate.bat @@ -0,0 +1,151 @@ +@echo off +goto Run +:Usage +echo.%~nx0 [flags and arguments] [quoted MSBuild options] +echo. +echo.Build CPython from the command line. Requires the appropriate +echo.version(s) of Microsoft Visual Studio to be installed (see readme.txt). +echo.Also requires Subversion (svn.exe) to be on PATH if the '-e' flag is +echo.given. +echo. +echo.After the flags recognized by this script, up to 9 arguments to be passed +echo.directly to MSBuild may be passed. If the argument contains an '=', the +echo.entire argument must be quoted (e.g. `%~nx0 "/p:PlatformToolset=v100"`) +echo. +echo.Available flags: +echo. -h Display this help message +echo. -V Display version information for the current build +echo. -r Target Rebuild instead of Build +echo. -d Set the configuration to Debug +echo. -e Build external libraries fetched by get_externals.bat +echo. Extension modules that depend on external libraries will not attempt +echo. to build if this flag is not present +echo. -m Enable parallel build (enabled by default) +echo. -M Disable parallel build +echo. -v Increased output messages +echo. -k Attempt to kill any running Pythons before building (usually done +echo. automatically by the pythoncore project) +echo. --pgo Build with Profile-Guided Optimization. This flag +echo. overrides -c and -d +echo. --test-marker Enable the test marker within the build. +echo. +echo.Available flags to avoid building certain modules. +echo.These flags have no effect if '-e' is not given: +echo. --no-ssl Do not attempt to build _ssl +echo. --no-tkinter Do not attempt to build Tkinter +echo. +echo.Available arguments: +echo. -c Release ^| Debug ^| PGInstrument ^| PGUpdate +echo. Set the configuration (default: Release) +echo. -p x64 ^| Win32 +echo. Set the platform (default: Win32) +echo. -t Build ^| Rebuild ^| Clean ^| CleanAll +echo. Set the target manually +echo. --pgo-job The job to use for PGO training; implies --pgo +echo. (default: "-m test --pgo") +exit /b 127 + +:Run +setlocal +set platf=Win32 +set vs_platf=x86 +set conf=Release +set target=Build +set dir=%~dp0 +set parallel=/m +set verbose=/nologo /v:m +set kill= +set do_pgo= +set pgo_job=-m test --pgo + +:CheckOpts +if "%~1"=="-h" goto Usage +if "%~1"=="-c" (set conf=%2) & shift & shift & goto CheckOpts +if "%~1"=="-p" (set platf=%2) & shift & shift & goto CheckOpts +if "%~1"=="-r" (set target=Rebuild) & shift & goto CheckOpts +if "%~1"=="-t" (set target=%2) & shift & shift & goto CheckOpts +if "%~1"=="-d" (set conf=Debug) & shift & goto CheckOpts +if "%~1"=="-m" (set parallel=/m) & shift & goto CheckOpts +if "%~1"=="-M" (set parallel=) & shift & goto CheckOpts +if "%~1"=="-v" (set verbose=/v:n) & shift & goto CheckOpts +if "%~1"=="-k" (set kill=true) & shift & goto CheckOpts +if "%~1"=="--pgo" (set do_pgo=true) & shift & goto CheckOpts +if "%~1"=="--pgo-job" (set do_pgo=true) & (set pgo_job=%~2) & shift & shift & goto CheckOpts +if "%~1"=="--test-marker" (set UseTestMarker=true) & shift & goto CheckOpts +if "%~1"=="-V" shift & goto Version +rem These use the actual property names used by MSBuild. We could just let +rem them in through the environment, but we specify them on the command line +rem anyway for visibility so set defaults after this +if "%~1"=="-e" (set IncludeExternals=true) & shift & goto CheckOpts +if "%~1"=="--no-ssl" (set IncludeSSL=false) & shift & goto CheckOpts +if "%~1"=="--no-tkinter" (set IncludeTkinter=false) & shift & goto CheckOpts + +if "%IncludeExternals%"=="" set IncludeExternals=false +if "%IncludeSSL%"=="" set IncludeSSL=true +if "%IncludeTkinter%"=="" set IncludeTkinter=true + +rem UBS has called get_externals_activestate.bat before running the build +rem if "%IncludeExternals%"=="true" call "%dir%get_externals_activestate.bat" + +if "%do_pgo%" EQU "true" if "%platf%" EQU "x64" ( + if "%PROCESSOR_ARCHITEW6432%" NEQ "AMD64" if "%PROCESSOR_ARCHITECTURE%" NEQ "AMD64" ( + echo.ERROR: Cannot cross-compile with PGO + echo. 32bit operating system detected. Ensure your PROCESSOR_ARCHITECTURE + echo. and PROCESSOR_ARCHITEW6432 environment variables are correct. + exit /b 1 + ) +) + +if "%platf%"=="x64" (set vs_platf=x86_amd64) + +rem Setup the environment +call "%dir%env.bat" %vs_platf% >nul + +if "%kill%"=="true" call :Kill + +if "%do_pgo%"=="true" ( + set conf=PGInstrument + call :Build + del /s "%dir%\*.pgc" + del /s "%dir%\..\Lib\*.pyc" + echo on + call "%dir%\..\python.bat" %pgo_job% + @echo off + call :Kill + set conf=PGUpdate +) +goto Build + +:Kill +echo on +msbuild "%dir%\pythoncore.vcxproj" /t:KillPython %verbose%^ + /p:Configuration=%conf% /p:Platform=%platf%^ + /p:KillPython=true + +@echo off +goto :eof + +:Build +rem Call on MSBuild to do the work, echo the command. +rem Passing %1-9 is not the preferred option, but argument parsing in +rem batch is, shall we say, "lackluster" +echo on +if "%IncludeSSL%"=="true" ( + if "%PERL%" == "" where perl > "%TEMP%\perl.loc" 2> nul && set /P PERL= <"%TEMP%\perl.loc" & del "%TEMP%\perl.loc" + if "%PERL%" == "" (echo Cannot locate perl.exe on PATH or as PERL variable & exit /b 4) + msbuild "%dir%openssl.vcxproj"^ + /p:Configuration=%conf% /p:Platform=%platf% +) +msbuild "%dir%pcbuild.proj" /t:%target% %parallel% %verbose%^ + /p:Configuration=%conf% /p:Platform=%platf%^ + /p:IncludeExternals=%IncludeExternals%^ + /p:IncludeSSL=%IncludeSSL% /p:IncludeTkinter=%IncludeTkinter%^ + /p:UseTestMarker=%UseTestMarker%^ + %1 %2 %3 %4 %5 %6 %7 %8 %9 + +@echo off +goto :eof + +:Version +rem Display the current build version information +msbuild "%dir%python.props" /t:ShowVersionInfo /v:m /nologo %1 %2 %3 %4 %5 %6 %7 %8 %9 diff --git a/PCbuild/get_externals_activestate.bat b/PCbuild/get_externals_activestate.bat new file mode 100644 index 00000000000000..c406434201e3be --- /dev/null +++ b/PCbuild/get_externals_activestate.bat @@ -0,0 +1,87 @@ +@echo off +setlocal +rem Simple script to fetch source for external libraries + +if not exist "%~dp0..\externals" mkdir "%~dp0..\externals" +pushd "%~dp0..\externals" + +echo.Fetching external libraries... + +set IncludeTkinterSrc=false +set IncludeSSLSrc=false + +rem *** IMPORTANT *** +rem If updating bzip2, db, nasm, openssl, or sqlite you must also edit their directory names in python.props. +rem If updating tcl/tk/tix you must also update their versions/directories in tcltk.props. + +set default_OpenSSL=1.1.1v +if "%OpenSSL_version%"=="" set OpenSSL_version=%default_OpenSSL% + +set libraries= +set libraries=%libraries% bzip2-1.0.8 +if NOT "%IncludeSSL%"=="false" set libraries=%libraries% nasm-2.11.06 +if NOT "%IncludeSSL%"=="false" set libraries=%libraries% openssl-%OpenSSL_version% +set libraries=%libraries% sqlite-3.43.0.0 +if NOT "%IncludeTkinter%"=="false" set libraries=%libraries% tcltk-8.6.9.0-bin +if NOT "%IncludeTkinter%"=="false" set libraries=%libraries% tix-8.4.3.6 +set libraries=%libraries% xz-5.2.2 +set libraries=%libraries% zlib-1.2.13 +rem CAMEL_GIT_SHA contains the commit SHA used for the current build. +rem Note that the branch 'master' in the below URL is ignored by the server because sha is present. + +for %%e in (%libraries%) do ( + if exist %%e ( + echo.%%e already exists, skipping. + ) else ( + echo.Fetching %%e... + call lwp-download https://s3.amazonaws.com/camel-sources/src/vendor-sources/python-core/%%e-pysvn.tar.gz ..\externals\%%e.tar.gz + cd ..\externals + call tar zxf %%e.tar.gz + del %%e.tar.gz + if exist cpython-source-deps-%%e ( + move cpython-source-deps-%%e %%e + ) + cd ..\PCbuild + ) +) + +cd ..\externals +if NOT "%OpenSSL_version%"=="%default_OpenSSL%" ( + if exist openssl-%OpenSSL_version% ( + move openssl-%OpenSSL_version% openssl-%default_OpenSSL% + ) +) +cd ..\PCbuild + +goto end + +:usage +echo.invalid argument: %1 +echo.usage: %~n0 [[ -c ^| --clean ] ^| --clean-only ] +echo. +echo.Pull all sources necessary for compiling optional extension modules +echo.that rely on external libraries. Requires svn.exe to be on your PATH +echo.and pulls sources from %SVNROOT%. +echo. +echo.Use the -c or --clean option to clean up all external library sources +echo.before pulling in the current versions. +echo. +echo.Use the --clean-only option to do the same cleaning, without pulling in +echo.anything new. +echo. +echo.Only the first argument is checked, all others are ignored. +echo. +echo.**WARNING**: the cleaning options unconditionally remove any directory +echo.that is a child of +echo. %CD% +echo.and matches wildcard patterns beginning with bzip2-, db-, nasm-, openssl-, +echo.tcl-, tcltk, tk-, tix-, sqlite-, or xz-, and as such has the potential +echo.to be very destructive if you are not aware of what it is doing. Use with +echo.caution! +popd +exit /b -1 + + +:end +echo Finished. +popd diff --git a/PCbuild/openssl.props b/PCbuild/openssl.props index a7e16793c7f283..03e1d7531653c4 100644 --- a/PCbuild/openssl.props +++ b/PCbuild/openssl.props @@ -5,7 +5,7 @@ $(opensslIncludeDir);%(AdditionalIncludeDirectories) - $(opensslOutDir);%(AdditionalLibraryDirectories) + $(opensslLibDir);%(AdditionalLibraryDirectories) ws2_32.lib;libcrypto.lib;libssl.lib;%(AdditionalDependencies) @@ -15,10 +15,10 @@ <_DLLSuffix Condition="$(Platform) == 'ARM64'">$(_DLLSuffix)-arm64 - <_SSLDLL Include="$(opensslOutDir)\libcrypto$(_DLLSuffix).dll" /> - <_SSLDLL Include="$(opensslOutDir)\libcrypto$(_DLLSuffix).pdb" /> - <_SSLDLL Include="$(opensslOutDir)\libssl$(_DLLSuffix).dll" /> - <_SSLDLL Include="$(opensslOutDir)\libssl$(_DLLSuffix).pdb" /> + <_SSLDLL Include="$(opensslLibDir)\libcrypto$(_DLLSuffix).dll" /> + <_SSLDLL Include="$(opensslLibDir)\libcrypto$(_DLLSuffix).pdb" /> + <_SSLDLL Include="$(opensslLibDir)\libssl$(_DLLSuffix).dll" /> + <_SSLDLL Include="$(opensslLibDir)\libssl$(_DLLSuffix).pdb" /> @@ -26,4 +26,4 @@ - \ No newline at end of file + diff --git a/PCbuild/pcbuild.proj b/PCbuild/pcbuild.proj index dbe30dd6a8a898..361fc64f2f0c3e 100644 --- a/PCbuild/pcbuild.proj +++ b/PCbuild/pcbuild.proj @@ -51,15 +51,15 @@ - - + + + - - - + + diff --git a/PCbuild/python.props b/PCbuild/python.props index 0938a9470cfb06..c28dc3197fe4c2 100644 --- a/PCbuild/python.props +++ b/PCbuild/python.props @@ -51,14 +51,36 @@ + $(SQLITE3_DIR) $(ExternalsDir)sqlite-3.43.0.0\ + $(sqlite3Dir)\ + $(BZIP2_DIR) $(ExternalsDir)bzip2-1.0.8\ - $(ExternalsDir)xz-5.2.2\ + $(bz2Dir)\ + $(LZMA_DIR) + $(ExternalsDir)xz-5.2.5\ + $(lzmaDir)\ + $(FFI_DIR) + $(ExternalsDir)libffi\$(ArchName)\ + $(libffiDir)\ + $(libffiDir)include + $(libffiDir) + $(OPENSSL_DIR) + + + $(ExternalsDir)openssl-bin-1.1.1v\$(ArchName)\ + $(opensslOutDir) + $(opensslOutDir)include $(ExternalsDir)openssl-1.1.1v\ - $(ExternalsDir)openssl-bin-1.1.1v\$(ArchName)\ - $(opensslOutDir)include + $(opensslDir)\ + $(opensslDir)include + $(opensslDir)lib $(ExternalsDir)\nasm-2.11.06\ + $(ZLIB_DIR) $(ExternalsDir)\zlib-1.2.13\ + $(zlibDir)\ + $(zlibDir)include + $(zlibDir)lib diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index bc398435bc2a3c..a3d5a267b98b73 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -50,7 +50,7 @@ true true - true + true false @@ -68,13 +68,16 @@ /Zm200 %(AdditionalOptions) - $(PySourcePath)Python;%(AdditionalIncludeDirectories) - $(zlibDir);%(AdditionalIncludeDirectories) - _USRDLL;Py_BUILD_CORE;Py_ENABLE_SHARED;MS_DLL_ID="$(SysWinVer)";%(PreprocessorDefinitions) - _Py_HAVE_ZLIB;%(PreprocessorDefinitions) - - - version.lib;shlwapi.lib;ws2_32.lib;%(AdditionalDependencies) + $(PySourcePath)Python;%(AdditionalIncludeDirectories);$(zlibIncludeDir) + $(zlibIncludeDir);%(AdditionalIncludeDirectories) + _USRDLL;Py_BUILD_CORE;Py_BUILD_CORE_BUILTIN;Py_ENABLE_SHARED;_Py_HAVE_ZLIB;MS_DLL_ID="$(SysWinVer)";%(PreprocessorDefinitions) + + + version.lib;shlwapi.lib;ws2_32.lib;pathcch.lib;zlib.lib;%(AdditionalDependencies) + $(OutDir);%(AdditionalLibraryDirectories);$(zlibLibDir) + $(OutDir);%(AdditionalLibraryDirectories);$(zlibLibDir) + $(OutDir);%(AdditionalLibraryDirectories);$(zlibLibDir) + $(OutDir);%(AdditionalLibraryDirectories);$(zlibLibDir) @@ -215,17 +218,17 @@ - - - - - - - - - - - + + + + + + + + + + + @@ -296,6 +299,7 @@ + @@ -407,20 +411,6 @@ - - - - - - - - - - - - - - diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index a86408be9f71a3..b105d896eb416f 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -1052,6 +1052,7 @@ Modules\zlib + diff --git a/PCbuild/tcltk.props b/PCbuild/tcltk.props index b185cb7b1e2818..0ed4c3762e04a6 100644 --- a/PCbuild/tcltk.props +++ b/PCbuild/tcltk.props @@ -2,23 +2,36 @@ - 8 - 6 - 9 - 0 + $(TCL_MAJOR_VERSION) + 8 + $(TCL_MINOR_VERSION) + 6 + $(TCL_PATCHLEVEL) + 12 + $(TCL_REVISION) + 0 $(TclMajorVersion) $(TclMinorVersion) $(TclPatchLevel) $(TclRevision) - 8 - 4 - 3 - 6 - $(ExternalsDir)tcl-core-$(TclMajorVersion).$(TclMinorVersion).$(TclPatchLevel).$(TclRevision)\ - $(ExternalsDir)tk-$(TkMajorVersion).$(TkMinorVersion).$(TkPatchLevel).$(TkRevision)\ - $(ExternalsDir)tix-$(TixMajorVersion).$(TixMinorVersion).$(TixPatchLevel).$(TixRevision)\ - $(ExternalsDir)tcltk-$(TclMajorVersion).$(TclMinorVersion).$(TclPatchLevel).$(TclRevision)\$(ArchName)\ - + $(TIX_MAJOR_VERSION) + 8 + $(TIX_MINOR_VERSION) + 4 + $(TIX_PATCHLEVEL) + 3 + $(TIX_REVISION) + 6 + $(TCL_DIR) + $(ExternalsDir)tcl-core-$(TclMajorVersion).$(TclMinorVersion).$(TclPatchLevel).$(TclRevision)\ + $(TK_DIR) + $(ExternalsDir)tk-$(TkMajorVersion).$(TkMinorVersion).$(TkPatchLevel).$(TkRevision)\ + $(TIX_DIR) + $(ExternalsDir)tix-$(TixMajorVersion).$(TixMinorVersion).$(TixPatchLevel).$(TixRevision)\ + $(TCL_TK_DIR) + $(ExternalsDir)tcltk-$(TclMajorVersion).$(TclMinorVersion).$(TclPatchLevel).$(TclRevision)\$(ArchName)\ + $(tcltkDir)\ + g tcl$(TclMajorVersion)$(TclMinorVersion)t$(TclDebugExt).dll tcl$(TclMajorVersion)$(TclMinorVersion)t$(TclDebugExt).lib tclsh$(TclMajorVersion)$(TclMinorVersion)t$(TclDebugExt).exe @@ -36,6 +49,7 @@ Release Debug $(BuildDirTop)_$(TclMachine) + $(BuildDirTop)_VC16 $(BuildDirTop)_VC13 $(BuildDirTop)_VC13 $(BuildDirTop)_VC12 diff --git a/Python/getcopyright.c b/Python/getcopyright.c index c1f1aad9b845b1..b9dfe27628d136 100644 --- a/Python/getcopyright.c +++ b/Python/getcopyright.c @@ -4,6 +4,7 @@ static const char cprt[] = "\ +Copyright (c) 2000-2023 ActiveState Software Inc.\n\ Copyright (c) 2001-2023 Python Software Foundation.\n\ All Rights Reserved.\n\ \n\ diff --git a/setup.py b/setup.py index bf90600eaad38a..38c487f2a8f3ec 100644 --- a/setup.py +++ b/setup.py @@ -44,6 +44,13 @@ def get_platform(): return sys.platform host_platform = get_platform() +# ActivePython change: +# ActivePython builds some external libraries into a private build +# directory for controlled use by common Python extension modules. +apy_prefix_dir = r"PT_CONFIG_build_prefix_abs_dir" +apy_inc_dir = os.path.join(apy_prefix_dir, "include") +apy_lib_dir = os.path.join(apy_prefix_dir, "lib") + # Were we compiled --with-pydebug or with #define Py_DEBUG? COMPILED_WITH_PYDEBUG = ('--with-pydebug' in sysconfig.get_config_var("CONFIG_ARGS")) @@ -1449,38 +1456,22 @@ class db_found(Exception): pass # # You can upgrade zlib to version 1.1.4 yourself by going to # http://www.gzip.org/zlib/ - zlib_inc = find_file('zlib.h', [], inc_dirs) - have_zlib = False - if zlib_inc is not None: - zlib_h = zlib_inc[0] + '/zlib.h' - version = '"0.0.0"' - version_req = '"1.1.3"' - if host_platform == 'darwin' and is_macosx_sdk_path(zlib_h): - zlib_h = os.path.join(macosx_sdk_root(), zlib_h[1:]) - with open(zlib_h) as fp: - while 1: - line = fp.readline() - if not line: - break - if line.startswith('#define ZLIB_VERSION'): - version = line.split()[2] - break - if version >= version_req: - if (self.compiler.find_library_file(lib_dirs, 'z')): - if host_platform == "darwin": - zlib_extra_link_args = ('-Wl,-search_paths_first',) - else: - zlib_extra_link_args = () - exts.append( Extension('zlib', ['zlibmodule.c'], - libraries = ['z'], - extra_link_args = zlib_extra_link_args)) - have_zlib = True - else: - missing.append('zlib') - else: - missing.append('zlib') + # + # ActivePython Change: + # ActivePython's build ALWAYS expects a zlib install in + # $(TOP)/build. + zlib_inc_dirs = [apy_inc_dir] + zlib_lib_dirs = [apy_lib_dir] + if sys.platform == "darwin": + zlib_extra_link_args = ('-Wl,-search_paths_first',) else: - missing.append('zlib') + zlib_extra_link_args = () + exts.append(Extension('zlib', ['zlibmodule.c'], + include_dirs = zlib_inc_dirs, + library_dirs = zlib_lib_dirs, + libraries = ['z'], + extra_link_args = zlib_extra_link_args)) + have_zlib = True # Helper module for various ascii-encoders. Uses zlib for an optimized # crc32 if we have it. Otherwise binascii uses its own. @@ -1495,6 +1486,8 @@ class db_found(Exception): pass exts.append( Extension('binascii', ['binascii.c'], extra_compile_args = extra_compile_args, libraries = libraries, + include_dirs = zlib_inc_dirs, + library_dirs = zlib_lib_dirs, extra_link_args = extra_link_args) ) # Gustavo Niemeyer's bz2 module. @@ -1893,6 +1886,8 @@ def detect_tkinter(self, inc_dirs, lib_dirs): include_dirs.append('/usr/X11/include') added_lib_dirs.append('/usr/X11/lib') + added_lib_dirs.insert(0, apy_lib_dir) + # If Cygwin, then verify that X is installed before proceeding if host_platform == 'cygwin': x11_inc = find_file('X11/Xlib.h', [], include_dirs)